gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-06 03:59:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into docs/scanner-heal-v2-plan

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
This commit is contained in:
houseme
2026-09-05 15:40:34 +08:00
committed by GitHub
parent 1a7532400d 2e4ab045b6
commit a46e030420
87 changed files with 11511 additions and 2253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/e2e-full-selection.txt
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
.config/ecstore-required-tests.json
+72
View File
@@ -0,0 +1,72 @@
{
"lane": "ci/test-and-lint",
"tests": [
{
"invariant": "write-quorum",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
},
{
"invariant": "metadata-rollback",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
},
{
"invariant": "stale-writer",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
},
{
"invariant": "range-body",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
},
{
"invariant": "multipart-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
},
{
"invariant": "list-uncommitted-version",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
},
{
"invariant": "minio-object-fixture",
"suite": "rustfs-filemeta",
"name": "filemeta::test::parses_real_minio_object_xlmeta"
},
{
"invariant": "corrupt-part-arrays",
"suite": "rustfs-filemeta",
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
}
],
"fixtures": [
{
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
}
]
}
.github/workflows/ci.yml
+7
View File
@@ -269,6 +269,7 @@ jobs:
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
run: |
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT
set +e
@@ -277,6 +278,12 @@ jobs:
--status-level all --final-status-level all \
2>&1 | tee artifacts/test-and-lint/nextest.log
status=${PIPESTATUS[0]}
if [[ "${status}" -eq 0 ]]; then
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
> artifacts/test-and-lint/core-test-listing.json \
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
&& test -s target/nextest/ci/junit.xml || status=$?
fi
{
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
echo "exit_status=${status}"
.github/workflows/rustfs-heal-test.yml
+56
View File
@@ -152,6 +152,60 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
STEPS_TABLE="/tmp/rustfs-heal-steps.md"
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
steps = {}
order = []
version = None
version_node = None
verdict = None
verdict_detail = ''
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = step_re.match(line)
if m:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
steps[n] = (desc, status) # later lines win (fail after pass)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
if m:
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Step Results\n\n')
if version:
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
out.write(f'- Version under test: **{version}**{node_note}\n')
if verdict:
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
out.write('\n')
out.write('| Step | Description | Result |\n')
out.write('| --- | --- | --- |\n')
for n in sorted(order, key=int):
desc, status = steps[n]
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
PY
{
echo "# RustFS heal test report"
echo ""
@@ -160,6 +214,8 @@ jobs:
echo "- Package: ${PACKAGE_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${STEPS_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
.github/workflows/rustfs-pool-expand-test.yml
+56
View File
@@ -380,6 +380,60 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md"
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
steps = {}
order = []
version = None
version_node = None
verdict = None
verdict_detail = ''
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = step_re.match(line)
if m:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
steps[n] = (desc, status) # later lines win (fail after pass)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
if m:
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Step Results\n\n')
if version:
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
out.write(f'- Version under test: **{version}**{node_note}\n')
if verdict:
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
out.write('\n')
out.write('| Step | Description | Result |\n')
out.write('| --- | --- | --- |\n')
for n in sorted(order, key=int):
desc, status = steps[n]
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
PY
{
echo "# RustFS pool expansion test report"
echo ""
@@ -389,6 +443,8 @@ jobs:
echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
echo ""
cat "${STEPS_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
.github/workflows/rustfs-upgrade-test.yml
+82 -35
View File
@@ -18,15 +18,15 @@ on:
workflow_dispatch:
inputs:
from_version:
description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
required: false
default: '1.0.0-rc.4-preview.1'
default: '1.0.0-rc.3'
from_url:
description: 'OLD .deb URL. Overrides from_version.'
required: false
type: string
to_version:
description: 'NEW RustFS release tag (leave empty for latest nightly)'
description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
required: false
to_url:
description: 'NEW .deb URL. Overrides to_version / nightly default.'
@@ -145,6 +145,7 @@ jobs:
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-upgrade-test.sh
@@ -175,6 +176,29 @@ jobs:
else
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
# Fail fast with a clear message when a requested release tag has
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
# letting the suite die mid-run on a 404.
check_release_asset() {
local version="$1" tag asset url
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
tag="${version#v}"
asset="rustfs_${tag//-/.}_amd64.deb"
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
echo " ${url}" >&2
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
exit 1
fi
echo "resolved ${tag} -> ${url}"
}
if [ -z "${FROM_URL}" ]; then
check_release_asset "${FROM_VERSION}"
fi
if [ -z "${TO_URL}" ]; then
check_release_asset "${TO_VERSION}"
fi
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
- name: Generate report
@@ -203,54 +227,75 @@ jobs:
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
topo_re = re.compile(
r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
rows = []
index = {}
topo_rows = []
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = topo_re.match(line)
if m:
topo_rows.append(m.groups())
continue
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
except FileNotFoundError:
rows = []
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
# Upgrade matrix: one row per topology/backend with the versions
# captured on the nodes (rustfs --version) and the aggregated
# result. The dashboard renders this table directly.
with open(matrix_file, 'w', encoding='utf-8') as out:
out.write('## Upgrade Matrix\n\n')
out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
out.write('| --- | --- | --- | --- | --- |\n')
for topo, backend, old_v, new_v, npass, nfail in topo_rows:
result = 'PASS' if nfail == '0' else 'FAIL'
out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
if not topo_rows:
out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
PY
{
echo "# RustFS upgrade compatibility report"
@@ -261,6 +306,8 @@ jobs:
echo "- To: ${TO_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${MATRIX_TABLE}" || true
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
Cargo.lock
Generated
+17 -16
View File
@@ -1660,7 +1660,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1679,7 +1679,7 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -2061,7 +2061,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"inout 0.1.4",
]
@@ -2531,7 +2531,7 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rand_core 0.6.4",
"subtle",
"zeroize",
@@ -2556,11 +2556,11 @@ dependencies = [
[[package]]
name = "crypto-common"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"typenum",
]
@@ -3852,7 +3852,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid 0.9.6",
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -4117,7 +4117,7 @@ dependencies = [
"crypto-bigint 0.5.5",
"digest 0.10.7",
"ff 0.13.1",
"generic-array 0.14.9",
"generic-array 0.14.7",
"group 0.13.0",
"hkdf 0.12.4",
"pem-rfc7468 0.7.0",
@@ -4562,9 +4562,9 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.14.9"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -4577,7 +4577,7 @@ version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rustversion",
"typenum",
]
@@ -5600,7 +5600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding 0.3.3",
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -7027,7 +7027,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"base64 0.21.7",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -9662,6 +9662,7 @@ dependencies = [
"async-trait",
"aws-credential-types",
"aws-sdk-s3",
"aws-smithy-async",
"aws-smithy-http-client",
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -11293,7 +11294,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
dependencies = [
"base16ct 0.2.0",
"der 0.7.10",
"generic-array 0.14.9",
"generic-array 0.14.7",
"pkcs8 0.10.2",
"subtle",
"zeroize",
@@ -12271,7 +12272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
Cargo.toml
+1
View File
@@ -243,6 +243,7 @@ aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.118.0" }
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
aws-sdk-sts = { default-features = false, version = "1.114.0" }
aws-smithy-async = { version = "1.3.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.16.0" }
aws-smithy-types = { version = "1.6.3" }
crates/ecstore/Cargo.toml
+1
View File
@@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
aws-smithy-async.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] }
crates/ecstore/src/api/mod.rs
+5 -3
View File
@@ -479,9 +479,11 @@ pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
};
}
crates/ecstore/src/bucket/bucket_target_sys.rs
+84 -25
View File
@@ -59,7 +59,7 @@ use rustfs_utils::http::{
insert_header,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::fmt;
use std::str::FromStr as _;
@@ -376,6 +376,11 @@ pub struct BucketTargetSys {
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
/// alongside `targets_map`, and read before it so an unreadable
/// configuration surfaces as a typed error instead of an empty target set.
unreadable_targets: Arc<RwLock<HashSet<String>>>,
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
pub hc_client: Arc<HttpClient>,
@@ -419,6 +424,7 @@ impl BucketTargetSys {
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
targets_map: Arc::new(RwLock::new(HashMap::new())),
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
h_mutex: Arc::new(RwLock::new(HashMap::new())),
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
hc_client: Arc::new(build_health_check_client()),
@@ -628,30 +634,40 @@ impl BucketTargetSys {
health_map.clone()
}
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> {
/// Targets of one bucket, or of every bucket when `bucket` is empty.
///
/// A bucket that simply has no targets yields an empty list; a bucket
/// whose persisted configuration cannot be decoded is an error, so an
/// admin listing reports the fault instead of an empty list that reads as
/// "replication is not configured" (rustfs/backlog#2282).
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result<Vec<BucketTarget>, BucketTargetError> {
let health_stats = self.target_health_stats().await;
let mut targets = Vec::new();
if !bucket.is_empty() {
if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await {
for mut target in bucket_targets.targets {
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
if let Some(health) = health_stats.get(&target.arn) {
target.total_downtime = health.offline_duration;
target.online = health.online;
target.last_online = health.last_online;
target.latency = target::LatencyStat {
curr: health.latency.curr,
avg: health.latency.avg,
max: health.latency.peak,
};
target.offline_count = health.offline_count;
match self.list_bucket_targets(bucket).await {
Ok(bucket_targets) => {
for mut target in bucket_targets.targets {
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
if let Some(health) = health_stats.get(&target.arn) {
target.total_downtime = health.offline_duration;
target.online = health.online;
target.last_online = health.last_online;
target.latency = target::LatencyStat {
curr: health.latency.curr,
avg: health.latency.avg,
max: health.latency.peak,
};
target.offline_count = health.offline_count;
}
targets.push(target);
}
targets.push(target);
}
}
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {}
Err(err) => return Err(err),
}
return targets;
return Ok(targets);
}
let targets_map = self.targets_map.read().await;
@@ -674,10 +690,16 @@ impl BucketTargetSys {
}
}
targets
Ok(targets)
}
pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
if self.unreadable_targets.read().await.contains(bucket) {
return Err(BucketTargetError::BucketRemoteTargetsUnreadable {
bucket: bucket.to_string(),
});
}
let targets_map = self.targets_map.read().await;
if let Some(targets) = targets_map.get(bucket) {
Ok(BucketTargets {
@@ -690,13 +712,30 @@ impl BucketTargetSys {
}
}
/// Record that this bucket's persisted targets configuration exists but
/// cannot be decoded (rustfs/backlog#2282).
///
/// Any snapshot published from an earlier readable load is deliberately
/// left in place: withdrawing it would produce exactly the silent "no
/// targets configured" state this marker exists to prevent. The marker is
/// cleared by the next successful publish, which is what makes a repaired
/// configuration take effect without a restart.
pub async fn mark_targets_unreadable(&self, bucket: &str) {
let update_mutex = self.target_update_mutex(bucket).await;
let _update_guard = update_mutex.lock().await;
self.unreadable_targets.write().await.insert(bucket.to_string());
}
pub async fn delete(&self, bucket: &str) {
let update_mutex = self.target_update_mutex(bucket).await;
let _update_guard = update_mutex.lock().await;
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
// then ssec_passthrough_map (always last; also taken standalone by the
// capability accessors).
// Lock order: unreadable_targets, then targets_map, then
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
// (always last; also taken standalone by the capability accessors).
self.unreadable_targets.write().await.remove(bucket);
let mut targets_map = self.targets_map.write().await;
let mut arn_remotes_map = self.arn_remotes_map.write().await;
let mut health_map = self.target_h_mutex.write().await;
@@ -1093,6 +1132,11 @@ impl BucketTargetSys {
/// Keeping persisted-config reads under the same mutex prevents a stale
/// reload from overwriting a concurrent credential rotation.
async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
// Reaching here means the persisted configuration decoded, so the
// unreadable marker (if any) is stale. Cleared before the maps below
// so `unreadable_targets` stays the outermost of this module's locks.
self.unreadable_targets.write().await.remove(bucket);
let mut clients = Vec::new();
if let Some(new_targets) = targets {
for target in &new_targets.targets {
@@ -1100,9 +1144,9 @@ impl BucketTargetSys {
}
}
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
// then ssec_passthrough_map (always last; also taken standalone by the
// capability accessors).
// Lock order: unreadable_targets (above), then targets_map, then
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
// (always last; also taken standalone by the capability accessors).
let mut targets_map = self.targets_map.write().await;
let mut arn_remotes_map = self.arn_remotes_map.write().await;
let mut health_map = self.target_h_mutex.write().await;
@@ -1161,6 +1205,11 @@ impl BucketTargetSys {
}
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
if meta.bucket_targets_unreadable() {
self.mark_targets_unreadable(bucket).await;
return;
}
let Some(config) = &meta.bucket_target_config else {
return;
};
@@ -2276,6 +2325,13 @@ pub enum BucketTargetError {
BucketRemoteTargetNotFound {
bucket: String,
},
/// The bucket's persisted targets configuration exists but cannot be
/// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the
/// bucket genuinely has no targets: callers must not degrade this one to
/// an empty target set (rustfs/backlog#2282).
BucketRemoteTargetsUnreadable {
bucket: String,
},
BucketRemoteArnTypeInvalid {
bucket: String,
},
@@ -2309,6 +2365,9 @@ impl fmt::Display for BucketTargetError {
BucketTargetError::BucketRemoteTargetNotFound { bucket } => {
write!(f, "Remote target not found for bucket: {bucket}")
}
BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => {
write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}")
}
BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => {
write!(f, "Invalid ARN type for bucket: {bucket}")
}
@@ -3256,7 +3315,7 @@ mod tests {
}],
);
let targets = sys.list_targets("", "").await;
let targets = sys.list_targets("", "").await.expect("listing every bucket's targets");
assert_eq!(targets.len(), 1);
assert!(!targets[0].online);
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+749 -33
View File
@@ -584,33 +584,173 @@ impl ExpiryOp for FreeVersionTask {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TransitionDeleteVersionPlan {
Direct { version_id_exact: bool },
ProbeLegacyUnknown,
}
fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
use rustfs_utils::http::metadata_compat::{
SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str,
};
if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) {
let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID);
if version_key_present {
if oi.transitioned_object.version_id.is_empty() {
let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| {
rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key)
.is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
&& !value.is_empty()
});
if !has_non_empty_version {
// MinIO writes the transitioned-versionID key with an empty value
// for unversioned tier objects. The backend probe remains the proof.
return Ok(true);
}
} else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID)
== Some(oi.transitioned_object.version_id.as_str())
{
return Ok(true);
}
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"legacy remote tier version metadata is conflicting or malformed",
));
}
if !oi.transitioned_object.version_id.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"legacy remote tier version metadata is missing or inconsistent",
));
}
return Ok(true);
}
let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object has conflicting transition version state metadata",
)
})?;
if persisted != oi.transition_version_state.as_str() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object transition version state metadata changed during decoding",
));
}
Ok(false)
}
fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result<TransitionDeleteVersionPlan, std::io::Error> {
match oi.transition_version_state {
rustfs_filemeta::TransitionVersionState::Unknown => {
if legacy_transition_version_state_missing(oi)? {
Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown)
} else {
validate_transition_remote_version(oi)
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact })
}
}
_ => validate_transition_remote_version(oi)
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }),
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ResolvedTransitionDeleteVersion {
version_id_exact: bool,
remote_already_missing: bool,
}
async fn acquire_free_version_tier_lease(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(TierOperationLease, bool), std::io::Error> {
let version_id_exact = validate_transition_remote_version(oi)?;
) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> {
let delete_plan = transition_remote_version_delete_plan(oi)?;
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
let lease =
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
.await
.map_err(std::io::Error::other)?;
Ok((lease, version_id_exact))
Ok((lease, delete_plan))
}
async fn resolve_transition_delete_version_plan(
oi: &ObjectInfo,
lease: &TierOperationLease,
delete_plan: TransitionDeleteVersionPlan,
) -> Result<ResolvedTransitionDeleteVersion, std::io::Error> {
match delete_plan {
TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion {
version_id_exact,
remote_already_missing: false,
}),
TransitionDeleteVersionPlan::ProbeLegacyUnknown => {
let expected_version = oi.transitioned_object.version_id.as_str();
if expected_version.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"remote tier cannot safely delete a legacy object without an exact version ID",
));
}
let probe = lease
.probe_transition_version(&oi.transitioned_object.name, expected_version)
.await?;
match (expected_version, probe) {
(expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual))
if expected == actual =>
{
lease.validate_remote_version_id(expected)?;
Ok(ResolvedTransitionDeleteVersion {
version_id_exact: true,
remote_already_missing: false,
})
}
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => {
Ok(ResolvedTransitionDeleteVersion {
version_id_exact: false,
remote_already_missing: true,
})
}
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot prove legacy transition delete state",
)),
_ => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"remote tier object version state is unknown",
)),
}
}
}
}
async fn execute_resolved_transition_delete(
oi: &ObjectInfo,
lease: &TierOperationLease,
resolved: ResolvedTransitionDeleteVersion,
) -> Result<(), std::io::Error> {
if !resolved.remote_already_missing {
delete_object_from_remote_tier_with_lease_idempotent(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
lease,
resolved.version_id_exact,
)
.await?;
}
Ok(())
}
async fn delete_free_version_remote_object_with_lease(
oi: &ObjectInfo,
lease: &TierOperationLease,
version_id_exact: bool,
delete_plan: TransitionDeleteVersionPlan,
) -> Result<(), std::io::Error> {
delete_object_from_remote_tier_with_lease_idempotent(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
lease,
version_id_exact,
)
.await?;
Ok(())
let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?;
execute_resolved_transition_delete(oi, lease, resolved).await
}
fn free_version_physical_topology_generation(api: &ECStore) -> String {
@@ -641,6 +781,16 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
{
let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?;
let expected_legacy_missing = legacy_transition_version_state_missing(expected)?;
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
&& expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
&& candidate_legacy_missing
&& expected_legacy_missing
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id
{
return Ok(true);
}
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version remote version state is unknown",
@@ -716,7 +866,7 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
.await
.map_err(std::io::Error::other)?;
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
let local_object = encode_dir_object(&oi.name);
let object_guards = api
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
@@ -734,16 +884,30 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
"tier free-version cleanup fence is invalid before remote delete",
));
}
let resolved = tokio::select! {
_ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
}
result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => {
result.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out")
})??
}
};
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"tier free-version cleanup fence changed after remote probe",
));
}
tokio::select! {
_ = cancel.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
}
result = tokio::time::timeout_at(
deadline,
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
) => {
result
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => {
result.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out")
})??;
}
}
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
@@ -791,8 +955,8 @@ async fn delete_free_version_remote_object(
oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await
}
#[allow(
@@ -808,8 +972,8 @@ where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?;
let result = delete_local().await;
drop(lease);
Ok(result)
@@ -4688,6 +4852,39 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TransitionReadVersionPlan {
Direct,
ProbeLegacyUnversioned,
}
const LEGACY_TRANSITION_READ_PROBE_TIMEOUT: StdDuration = StdDuration::from_secs(30);
fn transition_remote_version_read_plan(oi: &ObjectInfo) -> Result<TransitionReadVersionPlan, std::io::Error> {
let version = oi.transitioned_object.version_id.as_str();
match oi.transition_version_state {
rustfs_filemeta::TransitionVersionState::Unknown => {
if !legacy_transition_version_state_missing(oi)? {
return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct);
}
if version.is_empty() {
Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned)
} else {
Ok(TransitionReadVersionPlan::Direct)
}
}
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct),
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct),
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => {
Ok(TransitionReadVersionPlan::Direct)
}
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object version state conflicts with its version ID",
)),
}
}
// The resolver joins the tier manager as the second injected port this read
// needs; grouping the request half into a struct would churn every call site of
// a bug fix.
@@ -4702,7 +4899,12 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<GetObjectReader, std::io::Error> {
validate_transition_remote_version(oi)?;
let read_plan = transition_remote_version_read_plan(oi)?;
// Reject invalid ranges and encryption requests before a compatibility
// probe can amplify them into remote listing work.
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
.await
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
let lease = match expected_identity {
Some(identity) => {
@@ -4716,7 +4918,36 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
Err(err) => return Err(std::io::Error::other(err)),
};
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
match read_plan {
TransitionReadVersionPlan::Direct => {
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
}
TransitionReadVersionPlan::ProbeLegacyUnversioned => {
// RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing
// after an admin reconcile can persist every proven legacy state.
let probe = tokio::time::timeout(
LEGACY_TRANSITION_READ_PROBE_TIMEOUT,
tgt_client.probe_transition_candidate(&oi.transitioned_object.name),
)
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??;
match probe {
crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {}
crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => {
return Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot prove legacy unversioned transition state",
));
}
_ => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote tier object version state is unknown",
));
}
}
}
}
// The same read plan the local path uses, so the tier fetch is positioned in
// the object's *stored* coordinate system and the stream is handed the same
@@ -4724,9 +4955,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
// through a plaintext-coordinate range and skipping the transform is how a
// transitioned SSE object used to come back as silently corrupt bytes of the
// right length (rustfs/rustfs#6025).
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
.await
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
let mut gopts = WarmBackendGetOpts::default();
@@ -5599,11 +5827,13 @@ mod tests {
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
#[cfg(feature = "test-util")]
use crate::services::tier::test_util::MockWarmOp;
#[cfg(feature = "test-util")]
use crate::services::tier::test_util::register_mock_tier;
#[cfg(feature = "test-util")]
use crate::services::tier::tier::TierConfigMgr;
#[cfg(feature = "test-util")]
use crate::services::tier::warm_backend::WarmBackend as _;
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
@@ -6299,7 +6529,75 @@ mod tests {
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy transitioned object body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body.clone()),
i64::try_from(body.len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
let mut user_defined = HashMap::new();
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: i64::try_from(body.len()).expect("body length should fit"),
transitioned_object: TransitionedObject {
name: remote_object,
version_id: remote_version,
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec {
is_suffix_length: false,
start: 7,
end: 18,
});
let mut reader = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&range,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("legacy unknown state should still allow a non-destructive read");
let mut got = Vec::new();
reader
.stream
.read_to_end(&mut got)
.await
.expect("transitioned reader should drain");
assert_eq!(got, &body.as_ref()[7..=18]);
assert_eq!(backend.get_count().await, 1);
assert_eq!(backend.remove_count().await, 0);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"tier generation lease should release after EOF"
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
@@ -6315,6 +6613,181 @@ mod tests {
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(),
..Default::default()
};
let err = match get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
{
Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"),
Err(err) => err,
};
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
assert_eq!(backend.get_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
for persisted_version in [
Uuid::nil().to_string(),
"\u{fffd}".to_string(),
"bad\u{0001}version".to_string(),
] {
let mut user_defined = HashMap::new();
insert_legacy_transition_version_id(&mut user_defined, &persisted_version);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = match get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
{
Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"),
Err(err) => err,
};
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
backend.set_put_remote_version(Some(String::new())).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy unversioned transitioned object body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body.clone()),
i64::try_from(body.len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
assert!(remote_version.is_empty());
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: i64::try_from(body.len()).expect("body length should fit"),
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier: tier.clone(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(),
..Default::default()
};
let mut reader = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("probe-proven legacy unversioned state should allow a non-destructive read");
let mut got = Vec::new();
reader
.stream
.read_to_end(&mut got)
.await
.expect("transitioned reader should drain");
assert_eq!(got, body.as_ref());
assert_eq!(backend.remove_count().await, 0);
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Put {
object: remote_object.clone()
},
MockWarmOp::Probe {
object: remote_object.clone()
},
MockWarmOp::Get { object: remote_object },
]
);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"tier generation lease should release after EOF"
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() {
let manager = TierConfigMgr::new();
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&manager, &tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
"versioned-candidate".to_string(),
)))
.await;
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
tier,
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
..Default::default()
};
@@ -6330,19 +6803,28 @@ mod tests {
)
.await
{
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"),
Err(err) => err,
};
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]);
assert_eq!(backend.get_count().await, 0);
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
async fn free_version_delete_rejects_explicit_unknown_before_backend_io() {
let manager = TierConfigMgr::new();
let backend = register_mock_tier(&manager, "WARM").await;
let identity = test_tier_destination_identity(&manager, "WARM").await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
);
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
@@ -6351,17 +6833,251 @@ mod tests {
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect_err("unknown remote version state must fail before backend IO");
.expect_err("explicit unknown cleanup must fail before backend IO");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("version state is unknown"));
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
async fn test_tier_destination_identity(
manager: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
tier: &str,
) -> crate::services::tier::tier::TierDestinationId {
TierConfigMgr::acquire_operation_lease(manager, tier)
.await
.expect("test tier lease should be available")
.backend_identity()
}
#[cfg(feature = "test-util")]
fn user_defined_with_tier_destination_identity(
identity: crate::services::tier::tier::TierDestinationId,
) -> HashMap<String, String> {
let mut user_defined = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity),
);
user_defined
}
#[cfg(feature = "test-util")]
fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap<String, String> {
let mut user_defined = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
state.as_str().to_string(),
);
user_defined
}
#[cfg(feature = "test-util")]
fn insert_legacy_transition_version_id(user_defined: &mut HashMap<String, String>, version_id: &str) {
rustfs_utils::http::metadata_compat::insert_str(
user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
version_id.to_string(),
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() {
let manager = TierConfigMgr::new();
register_mock_tier(&manager, "WARM").await;
let identity = test_tier_destination_identity(&manager, "WARM").await;
let mut legacy_metadata = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version");
let mut explicit_metadata = legacy_metadata.clone();
rustfs_utils::http::metadata_compat::insert_str(
&mut explicit_metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
);
let make_info = |user_defined: HashMap<String, String>| ObjectInfo {
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: "legacy-version".to_string(),
tier: "WARM".to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata))
.expect_err("mixed legacy-missing and explicit unknown provenance must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy exact cleanup body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body),
i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
let mut user_defined = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing))
.await;
assert_eq!(
backend
.probe_transition_candidate_state(&remote_object)
.await
.expect("current remote view should be readable"),
TransitionCandidateProbe::Missing,
"a current delete marker must hide the historical data version from an unversioned probe"
);
backend.clear_op_log().await;
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: remote_version,
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("probe-proven legacy exact cleanup should delete the remote version");
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("a retry after the exact remote version is already missing should be idempotent");
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Get {
object: remote_object.clone()
},
MockWarmOp::Remove {
object: remote_object.clone()
},
MockWarmOp::Get {
object: remote_object.clone()
},
]
);
assert_eq!(
backend.remove_versions().await,
vec![(remote_object, object_info.transitioned_object.version_id)]
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_retains_legacy_unknown_unversioned_object() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
backend.set_put_remote_version(Some(String::new())).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
let body = Bytes::from_static(b"legacy unversioned cleanup body");
let remote_version = backend
.put(
&remote_object,
ReaderImpl::Body(body),
i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"),
)
.await
.expect("mock remote object should be stored");
assert!(remote_version.is_empty());
backend.clear_op_log().await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new());
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: String::new(),
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
let err = super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect_err("legacy unversioned cleanup cannot exclude a versioning-state race");
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
assert!(backend.op_log().await.is_empty());
assert_eq!(backend.remove_count().await, 0);
assert!(backend.remove_versions().await.is_empty());
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_delete_does_not_remove_a_different_remote_version() {
let manager = TierConfigMgr::new();
let tier = "WARM";
let backend = register_mock_tier(&manager, tier).await;
let identity = test_tier_destination_identity(&manager, tier).await;
let remote_object = format!("remote/{}", Uuid::new_v4());
backend.set_put_remote_version(Some("different-version".to_string())).await;
backend
.put(
&remote_object,
ReaderImpl::Body(Bytes::from_static(b"different remote version")),
i64::try_from(b"different remote version".len()).expect("body length should fit"),
)
.await
.expect("different remote version should be stored");
backend.clear_op_log().await;
let mut user_defined = user_defined_with_tier_destination_identity(identity);
insert_legacy_transition_version_id(&mut user_defined, "legacy-version");
let object_info = ObjectInfo {
transitioned_object: TransitionedObject {
name: remote_object.clone(),
version_id: "legacy-version".to_string(),
tier: tier.to_string(),
..Default::default()
},
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
user_defined: user_defined.into(),
..Default::default()
};
super::delete_free_version_remote_object(&object_info, &manager)
.await
.expect("a missing exact legacy version should be an idempotent cleanup success");
assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]);
assert_eq!(backend.remove_count().await, 0);
assert!(backend.remove_versions().await.is_empty());
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn free_version_remote_delete_requires_persisted_destination_identity() {
crates/ecstore/src/bucket/lifecycle/durable_namespace.rs
+234 -1
View File
@@ -25,6 +25,7 @@ use super::{
manual_transition_job, tier_delete_journal, transition_transaction,
};
use crate::error::{Error, Result};
use crate::services::tier::tier_probe_intent;
pub(crate) const ILM_META_PREFIX: &str = "ilm";
const ILM_META_OBJECT_PREFIX: &str = "ilm/";
@@ -35,6 +36,7 @@ pub(crate) enum DurableIlmRecordKind {
TierDeleteJournal,
TierDeleteDispatchManifest,
TransitionTransaction,
TierProbeIntent,
ManualTransitionJob,
ManualTransitionScope,
ManualTransitionTask,
@@ -73,6 +75,12 @@ pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = Durable
max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE,
kind: DurableIlmRecordKind::TransitionTransaction,
};
pub(crate) const TIER_PROBE_INTENT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "tier-probe-intent",
prefix: tier_probe_intent::TIER_PROBE_INTENT_RECORD_PREFIX,
max_record_size: tier_probe_intent::MAX_TIER_PROBE_INTENT_SIZE,
kind: DurableIlmRecordKind::TierProbeIntent,
};
pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "manual-transition-job",
prefix: "ilm/manual-transition/jobs",
@@ -98,11 +106,12 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
};
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
TIER_DELETE_JOURNAL_NAMESPACE,
TIER_DELETE_JOURNAL_V6_NAMESPACE,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
TRANSITION_TRANSACTION_NAMESPACE,
TIER_PROBE_INTENT_NAMESPACE,
MANUAL_TRANSITION_JOB_NAMESPACE,
MANUAL_TRANSITION_SCOPE_NAMESPACE,
MANUAL_TRANSITION_TASK_NAMESPACE,
@@ -200,6 +209,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
revision: u64,
state: transition_transaction::TransitionTransactionState,
},
TierProbeIntent {
content_sha256: String,
identity_sha256: String,
remote_version_sha256: String,
remote_version_known: bool,
owner_fence_sha256: String,
revision: u64,
state: tier_probe_intent::TierProbeIntentState,
},
ManualTransitionJob {
content_sha256: String,
identity_sha256: String,
@@ -232,6 +250,7 @@ impl DurableIlmRecordCheckpoint {
| Self::TierDeleteDispatchManifest { content_sha256, .. }
| Self::TierDeleteDispatchParent { content_sha256, .. }
| Self::TransitionTransaction { content_sha256, .. }
| Self::TierProbeIntent { content_sha256, .. }
| Self::ManualTransitionJob { content_sha256, .. }
| Self::ManualTransitionScope { content_sha256, .. }
| Self::ManualTransitionTask { content_sha256 }
@@ -421,6 +440,32 @@ impl DurableIlmRecordCheckpoint {
.is_some_and(|expected_revision| *next_revision == expected_revision)
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
}
(
Self::TierProbeIntent {
identity_sha256: previous_identity,
remote_version_sha256: previous_remote_version,
remote_version_known: previous_remote_version_known,
owner_fence_sha256: previous_owner_fence,
revision: previous_revision,
state: previous_state,
..
},
Self::TierProbeIntent {
identity_sha256: next_identity,
remote_version_sha256: next_remote_version,
owner_fence_sha256: next_owner_fence,
revision: next_revision,
state: next_state,
..
},
) => {
previous_identity == next_identity
&& previous_owner_fence == next_owner_fence
&& next_revision
.checked_sub(*previous_revision)
.is_some_and(|distance| distance == 1 && tier_probe_state_reaches(*previous_state, *next_state, distance))
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
}
(
Self::ManualTransitionJob {
content_sha256: previous_content,
@@ -500,6 +545,14 @@ impl DurableIlmRecordCheckpoint {
/// after the exact terminal ETag and terminal receipt were committed, to
/// purge older object versions exposed by that deletion.
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
if let Self::TierProbeIntent { state, .. } = terminal
&& !matches!(
state,
tier_probe_intent::TierProbeIntentState::AbortedNoRemote | tier_probe_intent::TierProbeIntentState::Completed
)
{
return false;
}
if self == terminal || self.validate_successor(terminal).is_ok() {
return true;
}
@@ -568,6 +621,37 @@ impl DurableIlmRecordCheckpoint {
}
})
}
(
Self::TierProbeIntent {
identity_sha256: previous_identity,
remote_version_sha256: previous_remote_version,
remote_version_known: previous_remote_version_known,
owner_fence_sha256: previous_owner_fence,
revision: previous_revision,
state: previous_state,
..
},
Self::TierProbeIntent {
identity_sha256: terminal_identity,
remote_version_sha256: terminal_remote_version,
owner_fence_sha256: terminal_owner_fence,
revision: terminal_revision,
state: terminal_state,
..
},
) => {
previous_identity == terminal_identity
&& previous_owner_fence == terminal_owner_fence
&& matches!(
terminal_state,
tier_probe_intent::TierProbeIntentState::AbortedNoRemote
| tier_probe_intent::TierProbeIntentState::Completed
)
&& terminal_revision
.checked_sub(*previous_revision)
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
}
_ => false,
}
}
@@ -606,6 +690,23 @@ fn transition_state_distance(
}
}
fn tier_probe_state_reaches(
from: tier_probe_intent::TierProbeIntentState,
to: tier_probe_intent::TierProbeIntentState,
revision_distance: u64,
) -> bool {
use tier_probe_intent::TierProbeIntentState::{AbortedNoRemote, CleanupPending, Completed, UploadOutcomeUnknown, Uploaded};
match (from, to) {
(UploadOutcomeUnknown, Uploaded | CleanupPending | AbortedNoRemote) => revision_distance == 1,
(UploadOutcomeUnknown, Completed) => matches!(revision_distance, 2 | 3),
(Uploaded, CleanupPending) => revision_distance == 1,
(Uploaded, Completed) => revision_distance == 2,
(CleanupPending, Completed) => revision_distance == 1,
_ => false,
}
}
fn manual_job_state_reaches(
from: manual_transition_job::ManualTransitionJobState,
to: manual_transition_job::ManualTransitionJobState,
@@ -1082,6 +1183,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
},
)
}
DurableIlmRecordKind::TierProbeIntent => {
let probe_id = tier_probe_intent::tier_probe_intent_id_from_record_object_name(path)
.map_err(|err| Error::other(err.to_string()))?;
let intent =
tier_probe_intent::TierProbeIntent::decode(probe_id, data).map_err(|err| Error::other(err.to_string()))?;
let canonical =
tier_probe_intent::tier_probe_intent_record_object_name(probe_id).map_err(|err| Error::other(err.to_string()))?;
if canonical != path {
return Err(Error::other("tier probe intent path is not canonical"));
}
let identity_sha256 = checkpoint_hash(&(
intent.probe_id,
&intent.operation,
&intent.tier_name,
intent.destination_id,
&intent.probe_object,
&intent.creator_id,
intent.creator_epoch,
intent.created_at_unix_nanos,
))?;
let remote_version_sha256 = checkpoint_hash(&intent.remote_version)?;
let owner_fence_sha256 = checkpoint_hash(&intent.owner)?;
(
"probe_id",
probe_id.to_string(),
DurableIlmRecordCheckpoint::TierProbeIntent {
content_sha256,
identity_sha256,
remote_version_sha256,
remote_version_known: !intent.remote_version.is_unknown(),
owner_fence_sha256,
revision: intent.revision,
state: intent.state,
},
)
}
DurableIlmRecordKind::ManualTransitionJob => {
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
.map_err(|err| Error::other(err.to_string()))?;
@@ -1237,6 +1374,102 @@ mod tests {
}
}
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
tier_probe_intent::TierProbeIntent {
probe_id,
revision: 1,
state: tier_probe_intent::TierProbeIntentState::UploadOutcomeUnknown,
operation: tier_probe_intent::TierProbeOperationIdentity::Verify {
config_etag: "config-etag".to_string(),
backend_identity: [1; 32],
},
tier_name: "COLD-A".to_string(),
destination_id: [1; 32],
probe_object: tier_probe_intent::tier_probe_object_name(probe_id),
creator_id: "node-a".to_string(),
creator_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture creator epoch should parse"),
created_at_unix_nanos: 1_780_000_000_000_000_000,
owner: tier_probe_intent::TierProbeOwnerFence {
owner_id: "node-a".to_string(),
owner_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture owner epoch should parse"),
not_after_unix_nanos: 1_780_000_900_000_000_000,
},
remote_version: tier_probe_intent::TierProbeRemoteVersion::default(),
}
}
fn tier_probe_checkpoint(intent: &tier_probe_intent::TierProbeIntent) -> DurableIlmRecordCheckpoint {
let path =
tier_probe_intent::tier_probe_intent_record_object_name(intent.probe_id).expect("tier probe path should build");
let encoded = intent.encode().expect("tier probe intent should encode");
let namespace = classify_durable_ilm_record(&path)
.expect("tier probe namespace should classify")
.expect("tier probe intent should be durable");
assert_eq!(namespace, &TIER_PROBE_INTENT_NAMESPACE);
validate_durable_ilm_record(&path, &encoded)
.expect("tier probe intent should validate")
.checkpoint
}
#[test]
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
let initial_intent = tier_probe_intent_fixture();
let initial = tier_probe_checkpoint(&initial_intent);
let mut uploaded_intent = initial_intent;
uploaded_intent
.advance(
tier_probe_intent::TierProbeIntentState::Uploaded,
tier_probe_intent::TierProbeRemoteVersion::versioned("opaque-v1"),
)
.expect("uploaded state should advance");
let uploaded = tier_probe_checkpoint(&uploaded_intent);
initial
.validate_successor(&uploaded)
.expect("durable receipt may adopt the exact uploaded generation");
let mut cleanup_intent = uploaded_intent.clone();
cleanup_intent
.advance(
tier_probe_intent::TierProbeIntentState::CleanupPending,
uploaded_intent.remote_version.clone(),
)
.expect("cleanup state should advance");
let cleanup = tier_probe_checkpoint(&cleanup_intent);
uploaded
.validate_successor(&cleanup)
.expect("durable receipt may adopt the exact cleanup generation");
let mut completed_intent = cleanup_intent.clone();
completed_intent
.advance(tier_probe_intent::TierProbeIntentState::Completed, cleanup_intent.remote_version.clone())
.expect("completed state should advance");
let completed = tier_probe_checkpoint(&completed_intent);
cleanup
.validate_successor(&completed)
.expect("durable receipt may adopt the exact terminal generation");
assert!(
initial.is_predecessor_of_terminal(&completed),
"terminal cleanup must recognize the full acknowledged-PUT path"
);
assert!(
initial.validate_successor(&completed).is_err(),
"ordinary receipt advancement must not skip intermediate generations"
);
assert!(
!initial.is_predecessor_of_terminal(&uploaded),
"a nonterminal generation must not be accepted as terminal proof"
);
let mut rebound = uploaded_intent;
rebound.owner.owner_epoch = Uuid::new_v4();
assert!(
rebound.encode().is_err(),
"dormant v1 must reject owner takeover before producing a checkpoint"
);
}
#[test]
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs
+5
View File
@@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
@@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent(
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+6
View File
@@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
let mut opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(observed_etag),
@@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag.to_string()),
..Default::default()
@@ -3780,6 +3783,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -3869,6 +3873,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
@@ -3893,6 +3898,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
+15 -15
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use super::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
@@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
});
#[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
#[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
#[derive(Debug)]
struct RemoteDeleteBreaker {
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
fn prune(&mut self, now: Instant) {
while let Some(ts) = self.failures.front().copied() {
if now.duration_since(ts) > self.window {
self.failures.pop_front();
let _ = self.failures.pop_front();
} else {
break;
}
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
return false;
}
if let Some(source) = err.get_ref() {
if error_chain_contains_signer_header_marker(source) {
return true;
}
if let Some(source) = err.get_ref()
&& error_chain_contains_signer_header_marker(source)
{
return true;
}
let message = err.to_string().to_ascii_lowercase();
@@ -205,7 +205,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
self.version_id = vid.clone();
self.version_id = vid;
self
}
@@ -219,7 +219,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts {
version_id: self.version_id.clone(),
version_id: self.version_id,
versioned: self.versioned,
version_suspended: self.suspended,
..Default::default()
@@ -388,8 +388,8 @@ impl Jentry {
impl ExpiryOp for Jentry {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.tier_name).as_bytes());
hasher.update(format!("{}", self.obj_name).as_bytes());
hasher.update(self.tier_name.as_bytes());
hasher.update(self.obj_name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
tier_name: &str,
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
.await
.map_err(std::io::Error::other)?;
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
crates/ecstore/src/bucket/lifecycle/transition_transaction.rs
+2
View File
@@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
crates/ecstore/src/bucket/metadata.rs
+162 -8
View File
@@ -477,6 +477,18 @@ impl BucketMetadata {
!self.table_bucket_config_json.is_empty()
}
/// `bucket-targets.json` is stored for this bucket but this build cannot
/// decode it.
///
/// Keeps "no replication targets configured" and "the target
/// configuration cannot be read" apart, the same distinction the
/// `fabricated` marker draws for the bucket metadata as a whole. Only
/// meaningful after [`Self::parse_all_configs`] has run; readers must fail
/// closed on `true` instead of serving an empty target set.
pub fn bucket_targets_unreadable(&self) -> bool {
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
}
/// Parsed per-bucket durability override, if a valid one is stored.
///
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
@@ -964,7 +976,32 @@ impl BucketMetadata {
Ok(())
}
fn parse_all_configs(&mut self) -> Result<()> {
/// Decode every stored sub-configuration into its typed field.
///
/// A decode failure never fails the whole load: this runs on every bucket
/// metadata read, including startup and peer reload, so one bucket's
/// corrupt sub-configuration must not make the bucket — or the node —
/// unloadable. Instead the failure is *retained*: the raw bytes stay
/// untouched and the typed field stays `None`, so `!raw.is_empty() &&
/// typed.is_none()` is the durable "exists but cannot be read" signal that
/// each accessor keys off. Which accessors must fail closed on it:
///
/// | Config | Verdict |
/// |---|---|
/// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
/// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
/// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
/// | replication | Fails closed in `get_replication_config`. |
/// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
/// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
/// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
/// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
/// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
/// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
/// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
if let Err(e) = self.parse_policy_config() {
tracing::warn!(
event = "bucket_metadata_parse_failed",
@@ -1088,20 +1125,26 @@ impl BucketMetadata {
"Failed to parse bucket metadata config"
);
}
// A stored targets blob that cannot be decoded must not collapse into
// the empty target set: that is indistinguishable from "no replication
// configured", so replication stops and no caller ever sees an error
// (rustfs/backlog#2282). Leaving the typed field `None` while the raw
// bytes stay non-empty is the retained parse failure every targets
// reader keys off; the bytes are preserved so the configuration is
// still recoverable.
self.bucket_target_config = None;
if !self.bucket_targets_config_json.is_empty() {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
.map(|t| self.bucket_target_config = Some(t))
{
tracing::warn!(
match serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) {
Ok(targets) => self.bucket_target_config = Some(targets),
Err(e) => tracing::error!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "bucket_targets",
error = %e,
"Failed to parse bucket metadata config"
);
self.bucket_target_config = Some(BucketTargets::default());
"Bucket replication targets are unreadable; replication for this bucket fails closed"
),
}
} else {
self.bucket_target_config = Some(BucketTargets::default());
@@ -1535,6 +1578,117 @@ mod test {
assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket");
}
/// rustfs/backlog#2282: a stored targets blob this build cannot decode
/// must not become the empty target set, and must stay distinguishable
/// from a bucket that never configured a target.
#[test]
fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() {
let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec();
let mut corrupt = BucketMetadata::new("corrupt-targets");
corrupt.bucket_targets_config_json = truncated.clone();
corrupt
.parse_all_configs()
.expect("one unreadable sub-config must not fail the whole metadata load");
assert!(
corrupt.bucket_target_config.is_none(),
"an undecodable targets blob must not produce a target set at all"
);
assert!(corrupt.bucket_targets_unreadable());
assert_eq!(
corrupt.bucket_targets_config_json, truncated,
"the raw bytes must survive so the configuration stays recoverable"
);
// The genuinely-absent case is unchanged, and the two now diverge.
let mut absent = BucketMetadata::new("no-targets");
absent.parse_all_configs().expect("absent targets parse");
assert!(
absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty),
"a bucket that configured no target still reads as an empty target set"
);
assert!(!absent.bucket_targets_unreadable());
}
/// `Credentials` carries no struct-level `serde(default)`, so one target
/// missing `secretKey` is a hard parse error for the whole document. That
/// must surface as "unreadable", never as "no targets configured".
#[test]
fn bucket_targets_missing_secret_key_are_unreadable_not_empty() {
let mut bm = BucketMetadata::new("missing-secret-key");
bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec();
bm.parse_all_configs()
.expect("a rejected targets document must not fail the whole metadata load");
assert!(
bm.bucket_targets_unreadable(),
"a targets document rejected for a missing secretKey is unreadable, not empty"
);
assert!(bm.bucket_target_config.is_none());
}
/// The invariant every branch of `parse_all_configs` shares: a stored but
/// undecodable payload keeps its raw bytes and leaves the typed field
/// `None`, so no branch fabricates a value. What a reader may then do with
/// that state is decided per config; see the table on `parse_all_configs`.
#[test]
fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() {
let malformed_xml = b"<not-a-valid-document".to_vec();
let malformed_json = b"{not-json".to_vec();
let mut bm = BucketMetadata::new("all-configs-malformed");
bm.policy_config_json = malformed_json.clone();
bm.quota_config_json = malformed_json.clone();
bm.bucket_targets_config_json = malformed_json.clone();
bm.notification_config_xml = malformed_xml.clone();
bm.lifecycle_config_xml = malformed_xml.clone();
bm.object_lock_config_xml = malformed_xml.clone();
bm.versioning_config_xml = malformed_xml.clone();
bm.encryption_config_xml = malformed_xml.clone();
bm.tagging_config_xml = malformed_xml.clone();
bm.replication_config_xml = malformed_xml.clone();
bm.cors_config_xml = malformed_xml.clone();
bm.logging_config_xml = malformed_xml.clone();
bm.website_config_xml = malformed_xml.clone();
bm.accelerate_config_xml = malformed_xml.clone();
bm.request_payment_config_xml = malformed_xml.clone();
bm.public_access_block_config_xml = malformed_xml.clone();
// `bucket_acl_config_json` is only checked for UTF-8, so only invalid
// UTF-8 exercises its failure branch.
bm.bucket_acl_config_json = vec![0xff, 0xfe];
bm.parse_all_configs()
.expect("a bucket whose every config is corrupt must still load its metadata");
let cleared: [(&str, bool); 17] = [
("policy", bm.policy_config.is_none()),
("quota", bm.quota_config.is_none()),
("bucket_targets", bm.bucket_target_config.is_none()),
("notification", bm.notification_config.is_none()),
("lifecycle", bm.lifecycle_config.is_none()),
("object_lock", bm.object_lock_config.is_none()),
("versioning", bm.versioning_config.is_none()),
("encryption", bm.sse_config.is_none()),
("tagging", bm.tagging_config.is_none()),
("replication", bm.replication_config.is_none()),
("cors", bm.cors_config.is_none()),
("logging", bm.logging_config.is_none()),
("website", bm.website_config.is_none()),
("accelerate", bm.accelerate_config.is_none()),
("request_payment", bm.request_payment_config.is_none()),
("public_access_block", bm.public_access_block_config.is_none()),
("bucket_acl", bm.bucket_acl_config.is_none()),
];
for (config, is_cleared) in cleared {
assert!(is_cleared, "{config}: a corrupt payload must not be replaced by a default");
}
assert_eq!(bm.bucket_targets_config_json, malformed_json, "raw bytes are retained");
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
}
#[test]
fn lifecycle_update_config_clears_parsed_config_on_delete() {
let mut bm = BucketMetadata::new("test-bucket");
crates/ecstore/src/bucket/metadata_sys.rs
+161 -4
View File
@@ -360,6 +360,16 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
}
async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
if bm.bucket_targets_unreadable() {
// "The configuration cannot be read" is not "no targets configured".
// Publishing an empty snapshot here is what silently stopped
// replication (rustfs/backlog#2282): mark the bucket instead, so every
// targets reader gets a typed error, and leave any snapshot from an
// earlier readable load in place rather than withdrawing it.
BucketTargetSys::get().mark_targets_unreadable(bucket).await;
return;
}
BucketTargetSys::get()
.update_all_targets(bucket, bm.bucket_target_config.as_ref())
.await;
@@ -2118,7 +2128,9 @@ impl BucketMetadataSys {
pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.public_access_block_config {
if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
Err(Error::other("persisted bucket public access block configuration is invalid"))
} else if let Some(config) = &bm.public_access_block_config {
Ok((config.clone(), bm.public_access_block_config_updated_at))
} else {
Err(Error::ConfigNotFound)
@@ -2429,7 +2441,9 @@ impl BucketMetadataSys {
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.sse_config {
if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
Err(Error::other("persisted bucket encryption configuration is invalid"))
} else if let Some(config) = &bm.sse_config {
Ok((config.clone(), bm.encryption_config_updated_at))
} else {
Err(Error::ConfigNotFound)
@@ -2500,7 +2514,9 @@ impl BucketMetadataSys {
pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.quota_config {
if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() {
Err(Error::other("persisted bucket quota configuration is invalid"))
} else if let Some(config) = &bm.quota_config {
Ok((config.clone(), bm.quota_config_updated_at))
} else {
Err(Error::ConfigNotFound)
@@ -2522,7 +2538,9 @@ impl BucketMetadataSys {
pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.bucket_target_config {
if bm.bucket_targets_unreadable() {
Err(Error::other("persisted bucket replication target configuration is invalid"))
} else if let Some(config) = &bm.bucket_target_config {
Ok(config.clone())
} else {
Err(Error::ConfigNotFound)
@@ -2593,6 +2611,7 @@ pub(crate) mod test_support {
mod tests {
use super::test_support::isolated_store_over_temp_disks;
use super::*;
use crate::bucket::bucket_target_sys::BucketTargetError;
use crate::bucket::metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
@@ -2788,6 +2807,36 @@ mod tests {
);
}
/// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor
/// whose configuration grants something — plaintext storage, anonymous
/// access, capacity, replication targets — reports a corrupt payload as
/// invalid rather than as absent, because "absent" is what grants it.
#[tokio::test]
async fn malformed_permissive_configs_are_not_reported_as_absent() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let bucket = "malformed-permissive-config";
let mut metadata = BucketMetadata::new(bucket);
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration".to_vec();
metadata.public_access_block_config_xml = b"<PublicAccessBlockConfiguration".to_vec();
metadata.quota_config_json = b"{not-json".to_vec();
metadata.bucket_targets_config_json = b"{not-json".to_vec();
metadata
.parse_all_configs()
.expect("a corrupt sub-config must not fail the load");
sys.set(bucket.to_string(), Arc::new(metadata)).await;
for (config, result) in [
("encryption", sys.get_sse_config(bucket).await.err()),
("public access block", sys.get_public_access_block_config(bucket).await.err()),
("quota", sys.get_quota_config(bucket).await.err()),
("bucket targets", sys.get_bucket_targets_config(bucket).await.err()),
] {
let err = result.unwrap_or_else(|| panic!("malformed {config} metadata must not read as a value"));
assert_ne!(err, Error::ConfigNotFound, "malformed {config} metadata must not be reported as absent");
}
}
#[tokio::test]
async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() {
use std::sync::atomic::Ordering;
@@ -4066,6 +4115,114 @@ mod tests {
target_sys.delete(bucket).await;
}
/// rustfs/backlog#2282: an unreadable `bucket-targets.json` reaches every
/// targets reader as a typed error; it neither withdraws a snapshot a
/// previous readable load published, nor collapses into the "no targets
/// configured" state that a bucket with an absent configuration reports.
#[tokio::test]
#[serial]
async fn unreadable_bucket_targets_fail_closed_and_stay_distinct_from_absent() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let target_sys = BucketTargetSys::get();
let unreadable = "targets-unreadable";
let absent = "targets-absent";
target_sys.delete(unreadable).await;
target_sys.delete(absent).await;
// A readable load publishes this bucket's targets.
let mut readable = BucketMetadata::new(unreadable);
readable.bucket_target_config = Some(BucketTargets {
targets: vec![target(unreadable, "live")],
});
sync_bucket_target_sys(unreadable, &readable).await;
assert_eq!(
target_sys
.list_bucket_targets(unreadable)
.await
.expect("readable targets publish")
.targets
.len(),
1
);
// The same bucket reloaded with a blob that cannot be decoded.
let mut corrupt = BucketMetadata::new(unreadable);
corrupt.bucket_targets_config_json = br#"{"targets":[{"endpoint":"#.to_vec();
corrupt
.parse_all_configs()
.expect("an unreadable targets blob must not fail the metadata load");
sys.set(unreadable.to_string(), Arc::new(corrupt)).await;
assert!(
matches!(
target_sys.list_bucket_targets(unreadable).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
),
"an unreadable configuration must not read as an empty or a missing target set"
);
assert!(
target_sys.list_targets(unreadable, "").await.is_err(),
"the admin listing must surface the fault instead of an empty list"
);
let err = sys
.get_bucket_targets_config(unreadable)
.await
.expect_err("an unreadable targets configuration must not read as a value");
assert_ne!(err, Error::ConfigNotFound, "unreadable must not be reported as absent");
// A bucket that never configured a target keeps its previous behavior.
let mut no_targets = BucketMetadata::new(absent);
no_targets.parse_all_configs().expect("absent targets parse");
sys.set(absent.to_string(), Arc::new(no_targets)).await;
assert!(
matches!(
target_sys.list_bucket_targets(absent).await,
Err(BucketTargetError::BucketRemoteTargetNotFound { .. })
),
"an absent configuration must still report as a missing target set"
);
assert!(
target_sys
.list_targets(absent, "")
.await
.expect("an absent configuration lists no targets")
.is_empty()
);
assert!(
sys.get_bucket_targets_config(absent)
.await
.expect("an absent targets configuration still reads as an empty set")
.is_empty(),
"the absent path must keep returning an empty target set, exactly as before"
);
// One bucket's unreadable configuration does not reach another bucket.
assert!(!matches!(
target_sys.list_bucket_targets(absent).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
));
// A repaired configuration takes effect on the next load, no restart.
let mut repaired = BucketMetadata::new(unreadable);
repaired.bucket_target_config = Some(BucketTargets {
targets: vec![target(unreadable, "repaired")],
});
sync_bucket_target_sys(unreadable, &repaired).await;
assert_eq!(
target_sys
.list_bucket_targets(unreadable)
.await
.expect("a repaired configuration clears the unreadable marker")
.targets
.len(),
1
);
target_sys.delete(unreadable).await;
target_sys.delete(absent).await;
}
#[tokio::test]
#[serial]
async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
crates/ecstore/src/bucket/on_demand_migration/backfill.rs
+1
View File
@@ -684,6 +684,7 @@ async fn write_checkpoint(
};
let opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(preconditions),
..Default::default()
};
crates/ecstore/src/bucket/on_demand_migration/breaker.rs
+6 -1
View File
@@ -86,7 +86,12 @@ impl BreakerVerdict {
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
BreakerVerdict::Failure
}
Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
Some(
SourceError::AccessDenied
| SourceError::Unsupported(_)
| SourceError::InvalidPagination(_)
| SourceError::Other(_),
) => BreakerVerdict::Neutral,
}
}
}
crates/ecstore/src/bucket/on_demand_migration/list_through.rs
+291 -20
View File
@@ -189,8 +189,8 @@ pub enum SourceListPlan {
/// delimiter — the source's own roll-up boundary matches the request's.
Page { prefix: String },
/// `filter.prefix` reaches past a delimiter, so every key the source could
/// contribute rolls into this one common prefix. One bounded probe listing
/// decides whether it exists; there is nothing to paginate.
/// contribute rolls into this one common prefix. Bounded probes follow
/// empty progressing pages until a key proves existence or the source ends.
Folded { probe_prefix: String, common_prefix: String },
}
@@ -279,6 +279,29 @@ pub struct FetchRequest {
pub token: Option<String>,
}
/// Invalid pagination metadata. Opaque cursor values are never included in errors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ListPageError {
#[error("truncated listing has no continuation token")]
Missing,
#[error("truncated listing has an empty continuation token")]
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
if is_truncated {
match next_token {
None => return Err(ListPageError::Missing),
Some("") => return Err(ListPageError::Empty),
Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
Some(_) => {}
}
}
Ok(())
}
#[derive(Debug, Default)]
struct SideState {
start: SideCursor,
@@ -364,6 +387,11 @@ impl ListThroughMerger {
/// or `filter.prefix` excludes it.
pub fn disable_source(&mut self) {
self.source.disabled = true;
// A refill can fail after a valid first page. A local-only response
// must discard both that source payload and its ordering horizon.
self.source.entries.clear();
self.source.pages.clear();
self.source.more = false;
}
pub fn next_fetch(&self) -> Option<FetchRequest> {
@@ -378,7 +406,13 @@ impl ListThroughMerger {
/// Records one fetched page. `entries` must be sorted by `name` and already
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
/// in the same order.
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
pub fn push_page(
&mut self,
side: MergeSide,
entries: Vec<ListEntryKey>,
is_truncated: bool,
next_token: Option<String>,
) -> Result<(), ListPageError> {
let state = match side {
MergeSide::Local => &mut self.local,
MergeSide::Source => &mut self.source,
@@ -387,15 +421,19 @@ impl ListThroughMerger {
Some(last) => last.next_token.clone(),
None => state.start.token.clone(),
};
// A truncated page without a cursor cannot be continued; treating the
// side as finished is the only alternative to looping on it forever.
state.more = is_truncated && next_token.is_some();
validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
// Also reject a cycle through an earlier page in this bounded fetch.
if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
return Err(ListPageError::Repeated);
}
state.more = is_truncated;
state.pages.push(FetchedPage {
token,
count: entries.len(),
next_token: is_truncated.then_some(next_token).flatten(),
});
state.entries.extend(entries);
Ok(())
}
pub fn finish(self) -> MergeOutcome {
@@ -599,9 +637,15 @@ mod tests {
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
merger.push_page(fetch.side, kept, truncated, next);
merger
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish();
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
}
page_sizes.push(outcome.picks.len());
for pick in &outcome.picks {
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
@@ -616,11 +660,25 @@ mod tests {
}
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
all.sort();
all.dedup();
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
entries
// This oracle builds the complete namespace independently of the
// provider's page/marker helper and the production merger.
let mut namespace = std::collections::BTreeMap::new();
for key in local.iter().chain(source) {
let Some(suffix) = key.strip_prefix(prefix) else {
continue;
};
if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
&& let Some((directory, _)) = suffix.split_once(delimiter)
{
namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
continue;
}
namespace.insert(key.clone(), false);
}
namespace
.into_iter()
.map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
.collect()
}
#[test]
@@ -662,7 +720,9 @@ mod tests {
token: None
})
);
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish();
assert_eq!(outcome.picks.len(), 1);
@@ -683,12 +743,14 @@ mod tests {
};
let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source();
merger.push_page(
MergeSide::Local,
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
true,
Some("local-2".to_string()),
);
merger
.push_page(
MergeSide::Local,
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
true,
Some("local-2".to_string()),
)
.expect("local cursor advances");
let outcome = merger.finish();
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token");
@@ -698,6 +760,212 @@ mod tests {
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
}
#[test]
fn truncated_pages_require_a_nonempty_advancing_cursor() {
for side in [MergeSide::Local, MergeSide::Source] {
for entries in [vec![], vec![ListEntryKey::object("a")]] {
for (next, expected) in [
(None, Err(ListPageError::Missing)),
(Some(""), Err(ListPageError::Empty)),
(Some("stuck"), Err(ListPageError::Repeated)),
(Some("advances"), Ok(())),
] {
let resume = ListThroughToken::new(
SideCursor {
token: Some("stuck".into()),
done: false,
},
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
let state = if side == MergeSide::Local {
&merger.local
} else {
&merger.source
};
assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
}
}
}
}
#[test]
fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
assert_eq!(
merger.next_fetch(),
Some(FetchRequest {
side: MergeSide::Source,
token: Some("stuck".into())
})
);
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
let mut merger = ListThroughMerger::new(2, None);
merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
for next in ["opaque-z", "opaque-a"] {
assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
merger
.push_page(MergeSide::Source, vec![], true, Some(next.into()))
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish();
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
assert_eq!(token.source.as_deref(), Some("opaque-a"));
let mut merger = ListThroughMerger::new(2, Some(&token));
assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("first".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("second".into()))
.expect("first page advances");
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
let mut merger = ListThroughMerger::new(2, None);
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
.expect("local EOF");
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
.expect("first source page advances");
assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Local,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn list_through_static_namespace_boundary_matrix() {
let corpus = [
"a",
"a/",
"a/b",
"a/b/child",
"a0",
"b",
"b/leaf",
"quote\"&<",
"space key",
"z",
"é",
"中/文",
];
for count in [0, 1, 3, 4, corpus.len()] {
let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
for placement in 0..3 {
let (local, source): (Vec<_>, Vec<_>) =
keys.iter()
.enumerate()
.fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
if placement != 1 || index % 2 == 0 {
local.push(key.clone());
}
if placement != 0 || index % 2 == 0 {
source.push(key.clone());
}
(local, source)
});
for prefix in ["", "a", "a/", "中/"] {
for delimiter in [None, Some("/")] {
for max_keys in [1, 3, 4] {
let oracle = expected(&local, &source, prefix, delimiter);
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
assert_eq!(
emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
oracle,
"count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
);
let expected_sizes: Vec<_> = if oracle.is_empty() {
vec![0]
} else {
oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
};
assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
}
}
}
}
}
}
#[test]
fn list_through_large_overlap_walk_keeps_all_5300_keys() {
let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
let (emitted, sizes) = walk(&local, &source, "", None, 333);
assert_eq!(emitted.len(), 5300);
for (index, (entry, side)) in emitted.iter().enumerate() {
assert_eq!(entry.name, format!("k{index:05}"));
assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
}
assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
}
#[test]
fn token_round_trips_and_rejects_tampering() {
let token = ListThroughToken::new(
@@ -796,7 +1064,10 @@ mod tests {
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(256))]
#![proptest_config(ProptestConfig {
rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
..ProptestConfig::with_cases(256)
})]
/// Full pagination of a merged listing equals the sorted, deduplicated
/// union of both sides, with every shared key served by local, and no
crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+308 -71
View File
@@ -14,9 +14,9 @@
//! Outbound client for an on-demand migration source bucket.
//!
//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared
//! remote builder and exposes the read-only surface the migration path
//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The
//! S3 backend uses the shared remote builder and exposes the surface the
//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! probe for admin validation). Every request carries the
//! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and
//! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of
@@ -25,6 +25,7 @@
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
//! forwarded: v1 rejects SSE-C source objects outright.
use super::list_through::{ListPageError, validate_list_page};
use crate::bucket::remote_s3_client::{
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
};
@@ -223,6 +224,8 @@ pub enum SourceError {
ServerError(u16),
#[error("unsupported source object: {0}")]
Unsupported(String),
#[error("invalid source listing: {0}")]
InvalidPagination(#[from] ListPageError),
#[error("source request failed: {0}")]
Other(String),
}
@@ -245,6 +248,7 @@ impl SourceError {
SourceError::Connect(_) => "connect",
SourceError::ServerError(_) => "server_error",
SourceError::Unsupported(_) => "unsupported",
SourceError::InvalidPagination(_) => "invalid_pagination",
SourceError::Other(_) => "other",
}
}
@@ -511,7 +515,7 @@ pub struct SourceObject {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourcePage {
pub objects: Vec<SourceObject>,
/// Rolled-up prefixes, in the local namespace; always empty when the
/// Rolled-up prefixes, in the same namespace as `objects`; always empty when the
/// request carried no delimiter.
pub common_prefixes: Vec<String>,
pub is_truncated: bool,
@@ -519,7 +523,8 @@ pub struct SourcePage {
}
/// One `ListObjectsV2` page request against the source. Keys are given in the
/// local namespace; `SourceClient` maps them through `source_prefix`.
/// local namespace at `SourceClient`, and in the source namespace at
/// `SourceBackend`; `SourceClient` maps them through `source_prefix`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourceListRequest<'a> {
pub prefix: Option<&'a str>,
@@ -575,8 +580,30 @@ impl Intercept for SourceProxyMarkerInterceptor {
}
}
pub struct SourceClient {
/// Read-only provider operations in the source bucket namespace.
///
/// Implementations must preserve streaming, honor the requested range and
/// pagination cursor, and classify failures without including credentials.
/// `SourceClient` owns prefix mapping so every provider shares the same local
/// namespace. Continuation tokens are opaque and must never be prefix-mapped.
#[async_trait::async_trait]
pub trait SourceBackend: Send + Sync {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError>;
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError>;
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError>;
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError>;
/// Verify bucket access; `SourceClient` separately probes a filtered listing.
async fn probe(&self) -> Result<(), SourceError>;
}
/// S3-compatible implementation, including request signing and anti-loop headers.
pub struct S3SourceBackend {
client: S3Client,
bucket: String,
}
pub struct SourceClient {
backend: Box<dyn SourceBackend>,
endpoint: String,
bucket: String,
source_prefix: Option<String>,
@@ -609,7 +636,10 @@ impl SourceClient {
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
Self {
client,
backend: Box::new(S3SourceBackend {
client,
bucket: spec.bucket.clone(),
}),
endpoint,
bucket: spec.bucket.clone(),
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
@@ -652,35 +682,15 @@ impl SourceClient {
}
pub async fn head_object(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
self.backend.head(&self.source_key(key)).await
}
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
/// Streams the object, preserving an optional HTTP byte range.
pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
self.backend.get(&self.source_key(key), range).await
}
/// Lists one page under the local `prefix`. Keys are returned in the
/// local namespace; entries outside `source_prefix` are skipped.
/// Lists one page under the local prefix.
pub async fn list_objects_v2(
&self,
prefix: Option<&str>,
@@ -696,9 +706,82 @@ impl SourceClient {
.await
}
/// [`Self::list_objects_v2`] with the delimiter and start-after the
/// list-through merge needs (rustfs/backlog#2164).
/// Maps keys and common prefixes while leaving opaque cursors untouched.
pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
let prefix = self.source_key(request.prefix.unwrap_or_default());
let start_after = request.start_after.map(|key| self.source_key(key));
let mut page = self
.backend
.list(&SourceListRequest {
prefix: Some(&prefix),
start_after: start_after.as_deref(),
..*request
})
.await?;
validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
page.objects = page
.objects
.into_iter()
.filter_map(|object| self.local_object(object))
.collect();
page.common_prefixes = page
.common_prefixes
.into_iter()
.filter_map(|prefix| self.local_key(&prefix).map(str::to_string))
.collect();
Ok(page)
}
fn local_object(&self, mut object: SourceObject) -> Option<SourceObject> {
object.key = self.local_key(&object.key)?.to_string();
Some(object)
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
self.backend.tagging(&self.source_key(key)).await
}
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
self.backend.probe().await?;
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
}
}
#[async_trait::async_trait]
impl SourceBackend for S3SourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
}
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
}
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
// `start_after` is silently ignored by S3 once a continuation token is
// present; refuse the ambiguous pair rather than list from the wrong
// position.
@@ -711,9 +794,9 @@ impl SourceClient {
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(self.source_key(request.prefix.unwrap_or_default()))
.prefix(request.prefix.unwrap_or_default())
.set_delimiter(request.delimiter.map(str::to_string))
.set_start_after(request.start_after.map(|after| self.source_key(after)))
.set_start_after(request.start_after.map(str::to_string))
.set_continuation_token(request.continuation_token.map(str::to_string))
.max_keys(request.max_keys)
.send()
@@ -722,22 +805,17 @@ impl SourceClient {
let is_truncated = output.is_truncated.unwrap_or(false);
let next_continuation_token = output.next_continuation_token;
if is_truncated && next_continuation_token.is_none() {
return Err(SourceError::Other(
"source reported a truncated listing without a continuation token".to_string(),
));
}
let objects = output
.contents
.unwrap_or_default()
.into_iter()
.filter_map(|object| self.source_object(object))
.filter_map(s3_source_object)
.collect();
let common_prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
.filter_map(|prefix| prefix.prefix)
.collect();
Ok(SourcePage {
@@ -748,48 +826,43 @@ impl SourceClient {
})
}
fn source_object(&self, object: SdkObject) -> Option<SourceObject> {
let key = self.local_key(object.key.as_deref()?)?.to_string();
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
let output = self
.client
.get_object_tagging()
.bucket(&self.bucket)
.key(self.source_key(key))
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect())
}
/// Admin validation: HeadBucket plus a one-key listing under the prefix.
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
async fn probe(&self) -> Result<(), SourceError> {
self.client
.head_bucket()
.bucket(&self.bucket)
.send()
.await
.map_err(classify_sdk_error)?;
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
Ok(())
}
}
fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
let key = object.key?;
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1194,6 +1267,70 @@ mod tests {
assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri);
}
#[tokio::test]
async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() {
let body = r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<IsTruncated>true</IsTruncated><NextContinuationToken>data/opaque</NextContinuationToken>
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
let next_body = body.replace("data/opaque", "data/next");
let (client, requests) =
scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
let first = client
.list_page(&SourceListRequest {
prefix: Some("photos/"),
delimiter: Some("/"),
start_after: Some("photos/a"),
max_keys: 2,
..Default::default()
})
.await
.expect("delimiter listing should succeed");
assert_eq!(first.common_prefixes, vec!["photos/"]);
assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque"));
let second = client
.list_page(&SourceListRequest {
continuation_token: first.next_continuation_token.as_deref(),
max_keys: 2,
..Default::default()
})
.await
.expect("opaque continuation should succeed");
assert_eq!(second.common_prefixes, first.common_prefixes);
let requests = recorded(&requests);
let query = |request: &RecordedRequest| {
Url::parse(&request.uri)
.expect("request URI")
.query_pairs()
.into_owned()
.collect::<HashMap<_, _>>()
};
let first_query = query(&requests[0]);
assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/"));
assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a"));
assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/"));
let second_query = query(&requests[1]);
assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque"));
assert!(!second_query.contains_key("start-after"));
}
#[tokio::test]
async fn list_page_rejects_ambiguous_cursor_before_sending() {
let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await;
let err = client
.list_page(&SourceListRequest {
start_after: Some("a"),
continuation_token: Some("opaque"),
max_keys: 1,
..Default::default()
})
.await
.expect_err("ambiguous list position must fail");
assert!(matches!(err, SourceError::Other(_)));
assert!(recorded(&requests).is_empty(), "invalid request must never reach the source");
}
#[tokio::test]
async fn list_objects_v2_rejects_truncated_page_without_token() {
let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await;
@@ -1201,7 +1338,104 @@ mod tests {
.list_objects_v2(None, None, 10)
.await
.expect_err("truncated page without token is corrupt");
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
}
#[tokio::test]
async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
for (truncated, next, expected) in [
(true, None, Some(ListPageError::Missing)),
(true, Some(""), Some(ListPageError::Empty)),
(true, Some("stuck"), Some(ListPageError::Repeated)),
(true, Some("opaque-next"), None),
(false, None, None),
(false, Some("stuck"), None),
] {
let next_xml = next
.map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
.unwrap_or_default();
let body = format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
);
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
let result = client
.list_page(&SourceListRequest {
continuation_token: Some("stuck"),
max_keys: 2,
..Default::default()
})
.await;
match expected {
Some(expected) => {
let error = result.expect_err("malformed pagination must fail at the provider boundary");
assert!(
matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
"{error:?}"
);
assert_eq!(error.class_label(), "invalid_pagination");
assert!(!error.is_retryable());
assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
}
None => {
let page = result.expect("progressing empty/nonempty pages and EOF are valid");
assert_eq!(page.is_truncated, truncated);
assert_eq!(page.next_continuation_token.as_deref(), next);
assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
if let Some(object) = page.objects.first() {
assert_eq!(object.key, "a");
}
}
}
let requests = recorded(&requests);
assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
assert!(requests[0].uri.contains("continuation-token=stuck"));
}
}
}
struct ListOnlyBackend(SourcePage);
#[async_trait::async_trait]
impl SourceBackend for ListOnlyBackend {
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
Ok(self.0.clone())
}
async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
panic!("unexpected HEAD in list test")
}
async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
panic!("unexpected GET in list test")
}
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
panic!("unexpected tagging in list test")
}
async fn probe(&self) -> Result<(), SourceError> {
panic!("unexpected probe in list test")
}
}
#[tokio::test]
async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
for (next, expected) in [
(None, ListPageError::Missing),
(Some(""), ListPageError::Empty),
(Some("stuck"), ListPageError::Repeated),
] {
let mut client = prefix_client(Some("data/".into()));
client.backend = Box::new(ListOnlyBackend(SourcePage {
is_truncated: true,
next_continuation_token: next.map(str::to_string),
..Default::default()
}));
let error = client
.list_objects_v2(None, Some("stuck"), 2)
.await
.expect_err("all providers must advance pagination");
assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
}
}
const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -1357,11 +1591,14 @@ mod tests {
fn prefix_client(prefix: Option<String>) -> SourceClient {
SourceClient {
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
backend: Box::new(S3SourceBackend {
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
bucket: "bucket".to_string(),
}),
endpoint: "https://source.example.com".to_string(),
bucket: "bucket".to_string(),
source_prefix: prefix.filter(|prefix| !prefix.is_empty()),
crates/ecstore/src/bucket/on_demand_migration/stats.rs
+1 -1
View File
@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
SourceError::Connect(_) => PullFailureReason::SourceConnect,
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
SourceError::Other(_) => PullFailureReason::SourceOther,
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
}
}
}
crates/ecstore/src/bucket/remote_s3_client.rs
+170 -1
View File
@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_async::time::TimeSource;
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
RemoteS3EndpointSpec {
@@ -824,6 +825,174 @@ mod tests {
);
}
#[derive(Clone, Debug)]
struct ClockSkewTimeSource(Arc<AtomicU64>);
impl TimeSource for ClockSkewTimeSource {
fn now(&self) -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
}
}
#[derive(Clone, Debug)]
struct ClockSkewConnector {
request_headers: RecordedHeaders,
error_code: &'static str,
skew_seconds: i64,
clock: ClockSkewTimeSource,
}
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
.unwrap_or_else(|| panic!("signed request must contain {name}"))
}
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
.expect("SDK signing timestamp must use the SigV4 format")
}
impl SmithyHttpConnector for ClockSkewConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
headers.push(
request
.headers()
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+ chrono::Duration::seconds(self.skew_seconds);
let (status, body) = if headers.len() == 1 {
(
403,
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
)
} else {
(200, String::new())
};
let response = http::Response::builder()
.status(status)
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
.header("content-type", "application/xml")
.header("content-length", body.len())
.body(SdkBody::from(body))
.expect("clock skew fixture response");
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
}
}
async fn clock_skew_client(
error_code: &'static str,
skew_seconds: i64,
retry: RemoteS3RetryPolicy,
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
let connector = SharedHttpConnector::new(ClockSkewConnector {
request_headers: Arc::clone(&headers),
error_code,
skew_seconds,
clock: clock.clone(),
});
let mut spec = spec("s3.example.com", true);
spec.retry = retry;
let config = build_remote_s3_config(&spec)
.await
.expect("clock skew fixture uses the production outbound configuration")
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
.time_source(clock.clone())
.build();
(S3Client::from_conf(config), headers, clock)
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for skew_seconds in [-600, 600] {
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect("clock skew GET must retry successfully");
assert_eq!(
headers.lock().expect("captured requests").len(),
2,
"{error_code}: GET needs exactly one retry"
);
clock.0.fetch_add(17, Ordering::SeqCst);
// SDK signing time is independent of Tokio's retry/scheduler clock.
tokio::time::advance(Duration::from_secs(61)).await;
client
.head_bucket()
.bucket("bucket")
.send()
.await
.expect("subsequent HEAD must use the client's cached skew");
let headers = headers.lock().expect("captured signed requests");
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
assert_eq!(
signing_time(&headers[1]),
initial + chrono::Duration::seconds(skew_seconds),
"{error_code}: retry must apply the measured offset exactly"
);
assert_eq!(
signing_time(&headers[2]),
initial + chrono::Duration::seconds(skew_seconds + 17),
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
);
let signature = |index: usize| {
recorded_header(&headers[index], "authorization")
.rsplit_once("Signature=")
.expect("SigV4 authorization contains a signature")
.1
};
assert_ne!(
signature(0),
signature(1),
"{error_code}: retry must be signed again after adjusting its date"
);
}
}
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for retry in [
RemoteS3RetryPolicy::Disabled,
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
] {
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
let error = client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect_err("clock skew must not override the caller's one-attempt budget");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
assert_eq!(
headers.lock().expect("captured requests").len(),
1,
"{error_code}: {retry:?} must send exactly one request"
);
}
}
}
#[test]
fn path_style_auto_and_path_force_path_style() {
assert!(PathStyle::Auto.force_path_style());
crates/ecstore/src/bucket/replication/replication_pool.rs
+18 -1
View File
@@ -46,7 +46,7 @@ use super::replication_storage_boundary::{
HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
ReplicationStorage,
};
use super::replication_target_boundary::{ReplicationTargetStore, replication_object_is_ssec_encrypted};
use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
use futures_util::stream::{self, StreamExt};
@@ -3084,6 +3084,23 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await {
Ok(targets) => Some(targets),
// A bucket whose persisted target configuration cannot be decoded has
// an unknown target set, not an empty one: scheduling against `None`
// here would drop every heal for it without a trace
// (rustfs/backlog#2282). Report it missed so the object is retried
// once the configuration is readable again.
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => {
warn!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
reason = "target_config_unreadable",
"Bucket replication targets are unreadable; replication heal queue fails closed"
);
return ReplicationQueueAdmission::Missed;
}
Err(err) => {
debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
crates/ecstore/src/bucket/replication/replication_target_boundary.rs
+2 -1
View File
@@ -15,7 +15,8 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError;
use crate::bucket::bucket_target_sys::BucketTargetSys;
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use http::HeaderMap;
crates/ecstore/src/bucket/sealed_credentials.rs
+12 -6
View File
@@ -28,7 +28,7 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::collections::BTreeMap;
use std::fmt;
use std::sync::{Arc, OnceLock};
@@ -81,8 +81,8 @@ impl SealScope {
/// The encryption context handed to the sealer. Keys are stable: they are
/// part of the on-disk contract, because a ciphertext only decrypts under
/// the same context.
pub fn encryption_context(&self) -> HashMap<String, String> {
HashMap::from([
pub fn encryption_context(&self) -> BTreeMap<String, String> {
BTreeMap::from([
("rustfs:store".to_string(), self.store.as_str().to_string()),
("rustfs:owner".to_string(), self.owner.clone()),
("rustfs:field".to_string(), self.field.to_string()),
@@ -201,12 +201,18 @@ pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Resu
mod tests {
use super::*;
use parking_lot::Mutex;
use std::collections::BTreeMap;
fn encode_context(context: &BTreeMap<String, String>) -> String {
let ordered = context.iter().collect::<BTreeMap<_, _>>();
serde_json::to_string(&ordered).expect("context serializes")
}
/// Stands in for the KMS-backed sealer: records the context it was called
/// with, and refuses a ciphertext presented under a different one.
#[derive(Default)]
struct FakeSealer {
sealed_contexts: Mutex<Vec<HashMap<String, String>>>,
sealed_contexts: Mutex<Vec<BTreeMap<String, String>>>,
}
#[async_trait]
@@ -214,7 +220,7 @@ mod tests {
async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
let context = scope.encryption_context();
self.sealed_contexts.lock().push(context.clone());
let mut bound = serde_json::to_string(&context).expect("context serializes");
let mut bound = encode_context(&context);
bound.push('|');
bound.push_str(plaintext);
Ok(SealedCredential {
@@ -231,7 +237,7 @@ mod tests {
.decode_to_vec(sealed.ct.as_bytes())
.map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes");
let expected = encode_context(&scope.encryption_context());
bound
.strip_prefix(&expected)
.and_then(|rest| rest.strip_prefix('|'))
crates/ecstore/src/core/pools.rs
+4
View File
@@ -5493,6 +5493,7 @@ where
fence.ensure_held()?;
let mut opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true,
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
..Default::default()
@@ -14412,6 +14413,7 @@ impl ECStore {
encoded.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -14566,6 +14568,7 @@ impl ECStore {
encoded,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(http_preconditions),
..Default::default()
},
@@ -14957,6 +14960,7 @@ impl ECStore {
encoded,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
crates/ecstore/src/disk/disk_store.rs
+77 -8
View File
@@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> {
self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
.await
.result
}
}
impl LocalDiskWrapper {
pub(in crate::disk) async fn rename_data_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> super::RenameDataObservation {
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
} else {
get_max_timeout_duration()
};
run_owned_mutation(external_guard, move || async move {
operation
let observed = run_owned_mutation(external_guard, move || async move {
let mut preflight_rejection = None;
let result = operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
operation
.disk
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
.await
// Preserve the former DiskAPI future's single boxing boundary.
let observed =
Box::pin(
operation
.disk
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
)
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
timeout_duration,
)
.await
.await;
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
preflight_rejection,
})
})
.await
.await;
observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
}
}
@@ -2588,6 +2617,46 @@ mod tests {
assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
}
#[tokio::test]
async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
for source_exists in [false, true] {
for guarded in [false, true] {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
.expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
if source_exists {
disk.make_volume("source").await.expect("source volume should exist");
}
let wrapper = LocalDiskWrapper::new(disk, false);
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
let mut file_info = FileInfo::new("object", 1, 0);
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
file_info.erasure.index = 1;
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
.await;
assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
.await;
assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.total_errors_availability, 1);
assert_eq!(snapshot.total_writes, 0);
}
}
}
#[tokio::test]
async fn local_disk_health_wrapper_counts_returned_availability_errors() {
let dir = tempfile::tempdir().expect("temp dir should be created");
crates/ecstore/src/disk/local.rs
+202 -1143
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/local/commit.rs
+1233
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/mod.rs
+49
View File
@@ -75,6 +75,25 @@ use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
pub(crate) struct RenameDataObservation {
pub(crate) result: Result<RenameDataResp>,
preflight_rejection: Option<local::LocalRenamePreflightRejection>,
}
impl RenameDataObservation {
fn unknown(result: Result<RenameDataResp>) -> Self {
Self {
result,
preflight_rejection: None,
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
}
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
@@ -711,6 +730,36 @@ impl Disk {
.await
}
pub(crate) async fn rename_data_borrowed_with_fence_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> RenameDataObservation {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.await,
),
}
}
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
crates/ecstore/src/object_api/types.rs
+16
View File
@@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink {
}
}
/// Internal PUT completion boundary; this does not change fsync or write quorum.
#[doc(hidden)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum WriteCompletion {
/// Return at write quorum when the commit owner can retain its guards.
#[default]
Quorum,
/// Drain the rename fan-out before returning. Minority failures still heal
/// after a successful quorum commit; this does not require every disk to succeed.
TailDrained,
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -896,6 +908,10 @@ pub struct ObjectOptions {
/// Persisted bucket incarnation observed before authorization.
pub expected_bucket_incarnation_id: Option<Uuid>,
pub no_lock: bool,
/// Control-plane writers that immediately read or CAS the same namespace
/// key use TailDrained without changing namespace lock ownership.
#[doc(hidden)]
pub write_completion: WriteCompletion,
/// True when an upper layer already holds the object read lock before
/// forwarding a no_lock read to the set layer.
pub metadata_cache_safe: bool,
crates/ecstore/src/services/notification_sys.rs
+483 -12
View File
@@ -62,12 +62,27 @@ const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
// Keep this synchronized with the version served by node_service. Including
// the local member in the minimum prevents an older coordinator from
// self-authorizing a policy implemented only by newer remote peers.
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
/// Version 5 is reserved for a fleet whose every metadata writer preserves
/// explicit transition version state and destination identity, and implements
/// conditional per-generation `xl.meta` writes with strong readback. The node
/// service must not advertise this version until the conditional writer from
/// rustfs/backlog#684 is available.
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
fn cross_pool_fence_policy_results(
peer_epochs: BTreeMap<String, Uuid>,
minimum_version: u32,
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
) -> (
CrossPoolFencePolicyResult,
CrossPoolFencePolicyResult,
CrossPoolFencePolicyResult,
CrossPoolFencePolicyResult,
) {
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
Ok(peer_epochs.clone())
} else {
@@ -78,7 +93,18 @@ fn cross_pool_fence_policy_results(
} else {
Err(Error::other("decommission target fence policy capability version is unsupported"))
};
(Ok(peer_epochs), journal_result, decommission_target_fence_result)
let legacy_transition_state_reconcile_result =
if minimum_version >= LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION {
Ok(peer_epochs.clone())
} else {
Err(Error::other("legacy transition state reconcile policy capability version is unsupported"))
};
(
Ok(peer_epochs),
journal_result,
decommission_target_fence_result,
legacy_transition_state_reconcile_result,
)
}
#[derive(Clone, Debug)]
@@ -252,10 +278,21 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
_permit: FleetCapabilityProofPermit,
}
/// Effect-window authority for one legacy transition-state reconciliation.
///
/// The token intentionally cannot be cloned. Its permit keeps the admitted
/// fleet generation alive until the caller finishes the final strong
/// readback, while revocation makes every later validation fail immediately.
pub struct LegacyTransitionStateReconcileFleetProofToken {
token: FleetCapabilityProofToken,
_permit: FleetCapabilityProofPermit,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -274,6 +311,10 @@ fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<Fl
DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
}
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
if let Some(proof) = state.proof.take() {
proof.generation.revoke();
@@ -444,6 +485,125 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
}
/// Acquire one non-cloneable authority that must span the complete reconcile
/// effect window, including its final strong readback.
pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let proof = {
let state = legacy_transition_state_reconcile_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, expected_topology, Instant::now())?
};
let observed_peer_epochs = observe_legacy_transition_state_reconcile_fleet(expected_topology).await?;
let state = legacy_transition_state_reconcile_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
&state,
&proof,
expected_topology,
&observed_peer_epochs,
Instant::now(),
)
.then_some(proof)
}
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
state: &FleetCapabilityProofState,
expected_topology: &str,
now: Instant,
) -> Option<LegacyTransitionStateReconcileFleetProofToken> {
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
let permit = state.proof.as_ref()?.generation.try_acquire()?;
Some(LegacyTransitionStateReconcileFleetProofToken { token, _permit: permit })
}
async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
let notification_sys = get_global_notification_sys()?;
let (peer_epochs, minimum_version) = timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_cross_pool_fence_fleet(expected_topology),
)
.await
.ok()?
.ok()?;
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peer_epochs, minimum_version);
reconcile_result.ok()
}
/// Revalidate the exact fleet generation captured by a reconcile token with a
/// fresh synchronous observation. Callers must await this before each
/// conditional metadata write and after the final strong readback.
pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
proof: &LegacyTransitionStateReconcileFleetProofToken,
) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
legacy_transition_state_reconcile_fleet_proof_matches_with_observer(
legacy_transition_state_reconcile_fleet_proof_slot(),
proof,
expected_topology,
|| observe_legacy_transition_state_reconcile_fleet(expected_topology),
)
.await
}
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
slot: &std::sync::RwLock<FleetCapabilityProofState>,
proof: &LegacyTransitionStateReconcileFleetProofToken,
expected_topology: &str,
observe: F,
) -> bool
where
F: FnOnce() -> Fut,
Fut: Future<Output = Option<BTreeMap<String, Uuid>>>,
{
{
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
if !legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
return false;
}
}
let Some(observed_peer_epochs) = observe().await else {
return false;
};
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
&state,
proof,
expected_topology,
&observed_peer_epochs,
Instant::now(),
)
}
fn legacy_transition_state_reconcile_fleet_proof_matches_at(
state: &FleetCapabilityProofState,
proof: &LegacyTransitionStateReconcileFleetProofToken,
expected_topology: &str,
now: Instant,
) -> bool {
proof._permit.generation.is_accepting()
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
&& state
.proof
.as_ref()
.is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
}
fn legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
state: &FleetCapabilityProofState,
proof: &LegacyTransitionStateReconcileFleetProofToken,
expected_topology: &str,
observed_peer_epochs: &BTreeMap<String, Uuid>,
now: Instant,
) -> bool {
legacy_transition_state_reconcile_fleet_proof_matches_at(state, proof, expected_topology, now)
&& proof.token.peer_epochs.as_ref() == observed_peer_epochs
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
let state = tier_delete_journal_fleet_proof_slot()
@@ -766,6 +926,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
cross_pool_fence_fleet_proof_slot(),
tier_delete_journal_fleet_proof_slot(),
decommission_target_fence_fleet_proof_slot(),
legacy_transition_state_reconcile_fleet_proof_slot(),
] {
mark_fleet_capability_topology_conflict(slot);
}
@@ -798,11 +959,12 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
};
let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
Err(err) => {
let message = err.to_string();
(
Err(Error::other(message.clone())),
Err(Error::other(message.clone())),
Err(Error::other(message.clone())),
Err(Error::other(message)),
@@ -818,6 +980,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
} else if let Some(err) = publish_fleet_capability_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
@@ -880,6 +1043,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
"notification capability probe"
);
}
if !topology_conflict
&& let Some(err) = publish_fleet_capability_probe_result(
legacy_transition_state_reconcile_fleet_proof_slot(),
&topology_fingerprint,
reconcile_result,
Instant::now(),
)
{
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "legacy_transition_state_reconcile_v1",
state = "failed_closed",
error = %err,
"notification capability probe"
);
}
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
}
});
@@ -959,7 +1140,7 @@ impl NotificationSys {
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
let mut minimum_version = u32::MAX;
let mut minimum_version = LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION;
for result in join_all(probes).await {
let (peer, version, epoch) = result?;
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
@@ -968,11 +1149,6 @@ impl NotificationSys {
minimum_version = minimum_version.min(version);
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
// A single-node deployment has no remote member to lower the local
// policy version advertised by this binary.
if minimum_version == u32::MAX {
minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
}
Ok((peer_epochs, minimum_version))
}
}
@@ -3190,20 +3366,36 @@ mod tests {
#[test]
fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
let (generic_v2, journal_v2, decommission_v2, reconcile_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
assert!(reconcile_v2.is_err(), "v2 cannot authorize legacy transition-state reconciliation");
let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
let (generic_v3, journal_v3, decommission_v3, reconcile_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
assert!(generic_v3.is_ok());
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
assert!(reconcile_v3.is_err());
let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
assert!(generic_v4.is_ok());
assert!(journal_v4.is_ok());
assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
assert!(
reconcile_v4.is_err(),
"the current local policy lacks the conditional xl.meta writer required by reconcile"
);
let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
assert!(generic_v5.is_ok());
assert!(journal_v5.is_ok());
assert!(decommission_v5.is_ok());
assert!(
reconcile_v5.is_ok(),
"only an all-v5 fleet preserves destination identity and conditional reconcile writes"
);
}
#[test]
@@ -3458,6 +3650,234 @@ mod tests {
);
}
#[test]
fn legacy_transition_state_reconcile_admits_only_compatible_single_and_multi_node_fleets() {
let now = Instant::now();
for peers in [
BTreeMap::new(),
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
] {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let (_, _, _, result) =
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", result, now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("an all-compatible fleet should admit reconciliation")
};
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now,
));
}
}
#[test]
fn legacy_transition_state_reconcile_restart_drains_concurrent_effect_windows() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let (_, _, _, original_result) =
cross_pool_fence_policy_results(original_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
let (first, second) = {
let state = slot.read().expect("reconcile proof slot should not poison");
(
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the first reconcile writer should be admitted"),
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the second reconcile writer should be admitted"),
)
};
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let (_, _, _, restarted_result) =
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
let blocked =
publish_fleet_capability_probe_result(&slot, "topology-a", restarted_result, now + Duration::from_millis(1))
.expect("a restarted member must revoke the old generation and wait for both writers");
assert!(blocked.to_string().contains("previous generation to drain"));
{
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(state.proof.is_none());
assert!(state.draining_generation.is_some());
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&first,
"topology-a",
now + Duration::from_millis(1),
));
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&second,
"topology-a",
now + Duration::from_millis(1),
));
}
drop(first);
let (_, _, _, still_blocked_result) =
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", still_blocked_result, now + Duration::from_millis(2),)
.is_some(),
"one remaining writer must keep the successor generation closed"
);
drop(second);
let (_, _, _, admitted_result) =
cross_pool_fence_policy_results(restarted_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", admitted_result, now + Duration::from_millis(3),)
.is_none(),
"the restarted generation may publish only after every old writer drains"
);
}
#[test]
fn legacy_transition_state_reconcile_fresh_observation_closes_the_polling_window() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let (_, _, _, original_result) =
cross_pool_fence_policy_results(original_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the original fleet should admit reconciliation")
};
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(
legacy_transition_state_reconcile_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
"the periodic cache has not observed the restart yet"
);
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
&state,
&admitted,
"topology-a",
&restarted_peers,
now,
));
let (_, _, _, downgraded) = cross_pool_fence_policy_results(original_peers, 4);
assert!(
downgraded.is_err(),
"a synchronous observation of a downgraded peer must fail before any cached proof can authorize a write"
);
}
#[tokio::test]
async fn legacy_transition_state_reconcile_invalid_token_skips_fleet_observation() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the original fleet should admit reconciliation")
};
revoke_fleet_capability_proof(&slot);
assert!(
!legacy_transition_state_reconcile_fleet_proof_matches_with_observer(&slot, &admitted, "topology-a", || async {
panic!("an invalid local generation must not trigger a fleet observation");
},)
.await
);
}
#[test]
fn legacy_transition_state_reconcile_membership_and_topology_changes_revoke_authority() {
let now = Instant::now();
for replacement in [
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]),
] {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let original = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the original fleet should admit reconciliation")
};
assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(replacement), now + Duration::from_millis(1),)
.is_some(),
"membership or process-epoch replacement must wait for the admitted writer"
);
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now + Duration::from_millis(1),
));
}
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(BTreeMap::new()), now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("the original topology should admit reconciliation")
};
mark_fleet_capability_topology_conflict(&slot);
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(state.topology_conflict);
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now,
));
}
#[test]
fn legacy_transition_state_reconcile_capability_downgrade_fails_closed() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
let now = Instant::now();
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
let (_, _, _, compatible_result) =
cross_pool_fence_policy_results(peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", compatible_result, now).is_none());
let admitted = {
let state = slot.read().expect("reconcile proof slot should not poison");
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
.expect("v5 should admit reconciliation")
};
let (_, _, _, downgraded_result) =
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION - 1);
let err = publish_fleet_capability_probe_result(&slot, "topology-a", downgraded_result, now + Duration::from_millis(1))
.expect("a v4 member must revoke reconcile authority");
assert!(err.to_string().contains("reconcile policy capability version is unsupported"));
let state = slot.read().expect("reconcile proof slot should not poison");
assert!(state.proof.is_none());
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
&state,
&admitted,
"topology-a",
now + Duration::from_millis(1),
));
assert!(
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now + Duration::from_millis(1),)
.is_none(),
"a downgraded fleet must remain inspect-only"
);
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
@@ -3539,6 +3959,57 @@ mod tests {
assert!(err.to_string().contains("incomplete"));
}
#[tokio::test]
async fn legacy_transition_state_reconcile_probe_rejects_missing_or_unreachable_members() {
let missing = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: vec![None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let missing_err = missing
.probe_cross_pool_fence_fleet("topology-a")
.await
.expect_err("a missing member slot must prevent reconcile capability proof");
assert!(missing_err.to_string().contains("incomplete"));
let unreachable = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let unreachable_err = unreachable
.probe_cross_pool_fence_fleet("topology-a")
.await
.expect_err("an unreachable member must prevent reconcile capability proof");
assert!(unreachable_err.to_string().contains("unreachable"));
}
#[tokio::test]
async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
let notification_sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: vec![None],
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let (peers, minimum_version) = notification_sys
.probe_cross_pool_fence_fleet("topology-a")
.await
.expect("a single-node capability probe should complete");
assert!(peers.is_empty());
assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
assert!(
reconcile_result.is_err(),
"the current node must not self-authorize reconcile before the conditional writer lands"
);
}
fn build_props(endpoint: &str) -> ServerProperties {
ServerProperties {
endpoint: endpoint.to_string(),
crates/ecstore/src/services/tier/mod.rs
+1
View File
@@ -21,6 +21,7 @@ pub mod tier_gen;
pub mod tier_handlers;
pub(crate) mod tier_mutation_intent;
pub mod tier_mutation_peer;
pub(crate) mod tier_probe_intent;
pub mod warm_backend;
pub mod warm_backend_aliyun;
pub mod warm_backend_azure;
crates/ecstore/src/services/tier/test_util.rs
+4 -1
View File
@@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend {
Ok(version)
}
async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.precondition().await?;
let barrier = self.inner.get_barrier.lock().await.take();
if let Some(barrier) = barrier {
@@ -719,6 +719,9 @@ impl WarmBackend for MockWarmBackend {
let Some(stored) = objects.get(object) else {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
};
if !rv.is_empty() && stored.remote_version_id != rv {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
}
let bytes = &stored.bytes;
let start = opts.start_offset.max(0) as usize;
crates/ecstore/src/services/tier/tier.rs
+1465 -132
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/services/tier/tier_admin.rs
+2 -4
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
@@ -145,7 +143,7 @@ mod tests {
assert_eq!(creds.access_key, "access");
assert_eq!(creds.secret_key, "secret");
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
assert_eq!(creds.creds_json.as_slice(), service_account);
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
assert_eq!(wire["access"], "access");
@@ -162,7 +160,7 @@ mod tests {
.expect("the former RustFS field names and byte-array encoding should remain readable");
assert_eq!(legacy.access_key, "legacy-access");
assert_eq!(legacy.secret_key, "legacy-secret");
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
assert_eq!(legacy.creds_json.as_slice(), service_account);
}
#[test]
crates/ecstore/src/services/tier/tier_mutation_intent.rs
+2
View File
@@ -460,6 +460,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -556,6 +557,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
crates/ecstore/src/services/tier/tier_probe_intent.rs
+1132
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/services/tier/warm_backend.rs
+52 -1
View File
@@ -40,6 +40,7 @@ use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_error_response::to_error_response,
api_put_object::{AdvancedPutOptions, PutObjectOptions},
transition_api::{ReadCloser, ReaderImpl},
};
@@ -48,11 +49,14 @@ use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
use s3s::header::{
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
X_AMZ_STORAGE_CLASS,
};
use s3s::{
S3ErrorCode,
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
@@ -141,6 +145,42 @@ pub trait WarmBackend {
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
Ok(TransitionCandidateProbe::Unsupported)
}
async fn probe_transition_version(
&self,
object: &str,
remote_version_id: &str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
if remote_version_id.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"an exact tier probe requires a remote version ID",
));
}
self.validate_remote_version_id(remote_version_id)?;
match self
.get(
object,
remote_version_id,
WarmBackendGetOpts {
start_offset: 0,
length: 1,
},
)
.await
{
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
}
Err(err)
if err.kind() == std::io::ErrorKind::NotFound
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
{
Ok(TransitionCandidateProbe::Missing)
}
Err(err) => Err(err),
}
}
async fn in_use(&self) -> Result<bool, std::io::Error>;
}
@@ -437,6 +477,17 @@ impl WarmBackend for MeteredWarmBackend {
Self::record(TierRequestOperation::Probe, result)
}
async fn probe_transition_version(
&self,
object: &str,
remote_version_id: &str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
Self::record(
TierRequestOperation::Probe,
self.inner.probe_transition_version(object, remote_version_id).await,
)
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
}
crates/ecstore/src/services/tier/warm_backend_aliyun.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_azure.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_gcs.rs
+8 -10
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::{HashMap, HashSet};
use std::future::Future;
@@ -146,11 +144,11 @@ pub struct WarmBackendGCS {
impl WarmBackendGCS {
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
if conf.creds == "" {
if conf.creds.is_empty() {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
if conf.bucket.is_empty() {
return Err(std::io::Error::other("no bucket name was provided"));
}
@@ -195,11 +193,11 @@ impl WarmBackendGCS {
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
if self.prefix.is_empty() {
object.to_string()
} else {
format!("{}/{}", self.prefix, object)
}
return dest_obj;
}
}
@@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS {
let bucket = gcs_bucket_resource_name(&self.bucket);
let Ok(res) = Box::pin(
self.client
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
.write_object(&bucket, self.get_dest(object), Bytes::from(d))
.send_buffered(),
)
.await
@@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let bucket = gcs_bucket_resource_name(&self.bucket);
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
let mut req = self.client.read_object(&bucket, self.get_dest(object));
let mut max_response_bytes = None;
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_minio.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_r2.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_rustfs.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_s3.rs
+42 -1
View File
@@ -529,6 +529,10 @@ mod tests {
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
];
let mut requests = Vec::new();
for response in responses {
@@ -622,15 +626,52 @@ mod tests {
.await
.expect_err("an authorization failure must not be mistaken for a missing key");
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
assert_eq!(
backend
.probe_transition_candidate("delete-marker-hidden")
.await
.expect("a current delete marker should hide the data version"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_version("delete-marker-hidden", "historical-version")
.await
.expect("the stored historical version should be probed exactly"),
TransitionCandidateProbe::VersionedPresent("historical-version".to_string())
);
assert_eq!(
backend
.probe_transition_version("delete-marker-hidden", "missing-version")
.await
.expect("a missing exact version should be classified"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_version("missing-object", "historical-version")
.await
.expect("a missing key for an exact version probe should be classified"),
TransitionCandidateProbe::Missing
);
let requests = fixture.await.expect("candidate fixture should join");
for request in requests {
for request in &requests[..6] {
let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
assert!(!request.contains("?versioning"));
assert!(!request.contains("?versions"));
}
for request in &requests[6..] {
let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
}
assert!(!requests[5].to_ascii_lowercase().contains("versionid="));
assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version"));
assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version"));
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
}
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
crates/ecstore/src/services/tier/warm_backend_tencent.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/set_disk/core/io_primitives.rs
+973 -158
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/mod.rs
+1 -1
View File
@@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(any(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))]
crates/ecstore/src/set_disk/ops/object.rs
+444 -31
View File
@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
};
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
#[cfg(test)]
use crate::storage_api_contracts::namespace::NamespaceLocking;
#[cfg(test)]
@@ -3548,6 +3548,7 @@ impl SetDisks {
(None, None, None)
};
let mut tmp_cleanup_owned = false;
let rollback_receipt = RenameRollbackReceipt::default();
let operation = async {
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
@@ -4256,6 +4257,7 @@ impl SetDisks {
let commit_bucket = bucket.to_owned();
let commit_object = object.to_owned();
let commit_tmp_dir = tmp_dir.clone();
let commit_rollback_receipt = rollback_receipt.clone();
let commit_object_lock_guard = object_lock_guard.take();
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
let commit_publication_guard = publication_commit_guard.take();
@@ -4266,13 +4268,17 @@ impl SetDisks {
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
// remote leases.
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& (commit_object_lock_guard.is_some()
|| commit_decommission_object_lock_guard.is_some()
|| commit_publication_guard.is_some())
let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
|| commit_decommission_object_lock_guard.is_some()
|| commit_publication_guard.is_some();
let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& commit_owns_namespace_guard
&& commit_scanner_publication_scope.is_none();
// Full-tail callers also transfer owned guards to the coordinator:
// cancelling their ACK waiter must not cancel an in-flight rename.
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|| commit_allows_early_ack
|| commit_owns_namespace_guard
|| commit_bucket_lifecycle_guard.is_some()
|| quota_mutation_fence;
let commit_write_path_label = write_path.metric_label();
@@ -4452,7 +4458,8 @@ impl SetDisks {
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
)
.with_publication_scope(commit_scanner_publication_scope.clone()),
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone()),
)
.await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
@@ -4585,6 +4592,11 @@ impl SetDisks {
let rename_commit = match rename_result {
Ok(commit) => commit,
Err(err) => {
if commit_rollback_receipt.is_incomplete() {
// Incomplete undo retains the staging source and
// rollback backup for recovery; cleanup is unsafe.
return Err(err.into());
}
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
} else if issue3031_diag_enabled() {
@@ -4617,9 +4629,8 @@ impl SetDisks {
request.object_version_id = committed_version_id
.or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string());
tokio::spawn(async move {
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
});
let heal_set = commit_set.clone();
tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
}
let rename_stage_elapsed = rename_stage_start.elapsed();
@@ -4885,7 +4896,7 @@ impl SetDisks {
);
}
});
} else {
} else if !rollback_receipt.is_incomplete() {
// Failure path (quorum loss / rollback): keep the cleanup inline so
// a failed PUT never returns while its tmp shards are still on disk
// (state-residue hardening tracked by backlog#864 / backlog#898).
@@ -17494,27 +17505,69 @@ mod put_object_tmp_cleanup_tests {
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_object_failure_cleans_tmp_workspace_inline() {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "tmp-clean-missing-bucket";
let object = "orphan-object";
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
let writer = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("missing-bucket PUT must stage before rename");
let staged = non_trash_tmp_entries(&temp_dirs).await;
assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
for workspace in staged {
let mut entries = tokio::fs::read_dir(&workspace)
.await
.expect("staged workspace should be readable");
let mut shards = 0;
while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
if entry.file_type().await.expect("staged entry type").is_dir() {
let part = tokio::fs::metadata(entry.path().join("part.1"))
.await
.expect("staging must contain an actual erasure shard");
assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
shards += 1;
}
}
assert_eq!(shards, 1);
}
assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("missing-bucket PUT must finish")
.expect("PUT task should join")
.expect_err("put_object into a missing bucket volume must fail");
assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
// The bucket volume is never created, so the shards are written into
// the tmp workspace and the commit fails at rename_data with a quorum
// error — exercising the failure-path cleanup.
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
let err = set_disks
.put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
.await
.expect_err("put_object into a missing bucket volume must fail");
// No polling: the failure path must clean the tmp workspace inline,
// before put_object returns (backlog#864 / backlog#898 hardening).
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
assert!(
leftovers.is_empty(),
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
);
drop(temp_dirs);
// No polling: known pre-publication rejection must clean staging
// inline, before PUT returns (backlog#864 / backlog#898).
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
assert!(
leftovers.is_empty(),
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
);
}
})
.await;
}
#[tokio::test]
@@ -18157,6 +18210,354 @@ mod put_object_tmp_cleanup_tests {
.await;
}
async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket)
.await
.expect("completion test bucket should be created");
}
}
/// Observe the actual metadata quorum while the remaining rename is parked.
/// A completed task count alone can race tasks that have not started yet.
async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let mut committed = 0;
for disk in disks {
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
Ok(_) => committed += 1,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
}
}
if committed == 3 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("three disks must publish metadata while the fourth rename remains paused");
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for size in [4096, 1024 * 1024] {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-cas";
let object = "full-tail-cas-object";
make_completion_test_bucket(&disks, bucket).await;
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("full-tail PUT must reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"the owned namespace guard must remain held"
);
barrier.release();
let written = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("full-tail PUT should finish after release")
.expect("full-tail PUT task should join")
.expect("full-tail PUT must commit");
assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("same-key lock should be available on return")
.expect("same-key lock probe should succeed"),
);
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("successful full-tail PUT must publish on every healthy disk");
}
drop(barrier);
let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
set.put_object(
bucket,
object,
&mut replacement,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: written.etag,
..Default::default()
}),
..Default::default()
},
)
.await
.expect("immediate same-key CAS must acquire the namespace guard");
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("CAS successor must be immediately readable");
let mut body = Vec::new();
read.stream.read_to_end(&mut body).await.expect("successor body must drain");
assert_eq!(body, b"cas successor");
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-heal";
let object = "full-tail-heal-object";
make_completion_test_bucket(&disks, bucket).await;
let mut heals = set.capture_test_rename_tail_heals();
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("failed tail must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
barrier.release();
tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("failed tail should drain")
.expect("PUT task should join")
.expect("a minority tail error must not negate committed quorum");
assert_eq!(tasks.running(), 0);
let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
.await
.expect("failed tail must schedule heal")
.expect("heal capture must remain connected");
assert_eq!(heal.bucket, bucket);
assert_eq!(heal.object_prefix.as_deref(), Some(object));
let info = set
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("committed object must remain readable despite the failed tail");
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_rejects_quorum_minus_one() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-no-quorum";
let object = "full-tail-no-quorum-object";
make_completion_test_bucket(&disks, bucket).await;
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
let tasks = rename_fanout_barrier::observe_tasks(object);
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
let err = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect_err("draining two successful disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
assert!(
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
"failed fresh write must not become visible"
);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
use crate::set_disk::core::io_primitives::rollback_fault_injection;
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
for fault in [
rollback_fault_injection::Fault::Io,
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
] {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-incomplete-undo";
let object = "incomplete-undo-object";
make_completion_test_bucket(&disks, bucket).await;
let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
set.put_object(
bucket,
object,
&mut old_reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("old generation should be completely committed");
wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
let old = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("old metadata must be readable");
let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("overwrite must enter the actual rename fan-out before failure injection");
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("incomplete undo must return without hanging")
.expect("PUT task should join")
.expect_err("two renamed disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
let leftovers = non_trash_tmp_entries(&dirs).await;
assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
let backups = dirs
.iter()
.filter(|dir| {
dir.path()
.join(bucket)
.join(object)
.join(old_data_dir.to_string())
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
.exists()
})
.count();
assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
// The remaining three disks still serve the old generation;
// the failed minority must never become an acknowledged write.
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("old generation must remain readable after incomplete rollback");
let mut body = Vec::new();
read.stream
.read_to_end(&mut body)
.await
.expect("old generation should stream");
assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = RUSTFS_META_BUCKET;
let object = "full-tail-cancelled-receipt";
// Internal config writes do not own a bucket lifecycle guard. The object
// guard alone must keep the full-tail coordinator alive after cancellation.
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("cancelled receipt must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
put.abort();
assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"owned coordinator must retain the namespace guard after waiter cancellation"
);
barrier.release();
drop(
tokio::time::timeout(Duration::from_secs(30), lock_probe)
.await
.expect("cancelled coordinator must eventually release its guard")
.expect("post-commit lock probe should succeed"),
);
assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("caller cancellation must not interrupt committed receipt materialization");
}
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
@@ -18184,6 +18585,7 @@ mod put_object_tmp_cleanup_tests {
&mut reader,
&ObjectOptions {
no_lock: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
@@ -18209,7 +18611,18 @@ mod put_object_tmp_cleanup_tests {
put.await
.expect("no-lock PUT task should join")
.expect("no-lock PUT should commit after the rename tail releases");
let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"full-tail PUT must not release the caller's outer guard"
);
drop(outer_guard);
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("outer owner releasing its guard should unblock the probe")
.expect("post-outer-guard probe should succeed"),
);
})
.await;
}
crates/ecstore/src/set_disk/transition_matrix_tests.rs
+39 -15
View File
@@ -18,6 +18,7 @@ use super::{
};
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
use crate::ecstore_validation_blackbox::make_local_set_disks;
use crate::object_api::WriteCompletion;
use crate::services::tier::test_util::register_mock_tier;
use crate::storage_api_contracts::bucket::BucketOperations;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -25,19 +26,24 @@ use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
use tokio::io::AsyncReadExt;
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
set_disks
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
.await
.expect("object metadata should resolve");
let generation = set_disks
.get_object_metadata_cache_generation(bucket, object)
.expect("metadata generation should be active");
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
assert!(
set_disks.get_object_metadata_cache.get(&key).await.is_some(),
"metadata read should publish the generation under test"
);
key
tokio::time::timeout(Duration::from_secs(30), async {
loop {
set_disks
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
.await
.expect("object metadata should resolve");
let generation = set_disks
.get_object_metadata_cache_generation(bucket, object)
.expect("metadata generation should be active");
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
return key;
}
tokio::task::yield_now().await;
}
})
.await
.expect("metadata read should publish the generation under test")
}
async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
@@ -60,8 +66,17 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
let original = set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("source object should be written");
let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;
@@ -164,8 +179,17 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
let original = set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("source object should be written");
crates/ecstore/src/store/init.rs
+367 -29
View File
@@ -864,6 +864,11 @@ mod tests {
save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
},
tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
tier_probe_intent::{
TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
},
warm_backend::{TransitionCandidateProbe, WarmBackend},
},
set_disk::SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
@@ -2974,6 +2979,33 @@ mod tests {
#[cfg(feature = "test-util")]
const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
fn decommission_retry_fault_hook(
bucket: &str,
object: &str,
faults: Arc<AtomicUsize>,
) -> crate::core::pools::DecommissionTestFaultDecision {
let target_bucket = bucket.to_string();
let target_object = object.to_string();
Arc::new(move |stage, bucket, object, _attempt, succeeded| {
if !succeeded
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|| bucket != target_bucket
|| object != target_object
{
return false;
}
// Entry retries reset the local attempt; real copy errors can skip
// successful attempts. Only injected faults spend this global budget.
faults
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
.then_some(faults.saturating_add(1))
})
.is_ok()
})
}
async fn seed_decommission_source(
store: &Arc<crate::store::ECStore>,
bucket: &str,
@@ -5115,6 +5147,33 @@ mod tests {
shutdown.cancel();
}
#[test]
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
let faults = Arc::new(AtomicUsize::new(0));
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
for (stage, bucket, object, succeeded) in [
("other-stage", "bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
] {
assert!(!hook(stage, bucket, object, 1, succeeded));
}
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
for (index, attempt) in attempts.into_iter().enumerate() {
assert_eq!(
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
index < 2,
"attempts={attempts:?}, index={index}"
);
}
assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
}
}
#[test]
#[serial_test::serial(storage_class_env)]
fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
@@ -5209,31 +5268,8 @@ mod tests {
));
let ordinary_faults = Arc::new(AtomicUsize::new(0));
let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
let fault_bucket = other_bucket.clone();
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
move |stage, bucket, object, attempt, succeeded| {
let candidate = succeeded
&& stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
&& bucket == fault_bucket.as_str()
&& object == other_object;
if !candidate {
return false;
}
// Keep the fault budget global across any
// entry-level re-list; its inner attempt counter
// restarts after SourceChanged.
ordinary_faults_for_hook
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
let next_fault = faults.saturating_add(1);
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
&& attempt == next_fault)
.then_some(next_fault)
})
.is_ok()
},
));
let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
let rx = CancellationToken::new();
let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
@@ -8040,10 +8076,15 @@ mod tests {
);
assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone())
let full_tail = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
.await
.expect("second page receipt should restore");
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec())
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
.await
.expect("second page receipt should corrupt deterministically");
let corrupt = store
@@ -11570,6 +11611,7 @@ mod tests {
pool_index: usize,
bucket: &str,
object: &str,
minio_unversioned: bool,
) {
for disk_index in 0..4 {
let metadata_path =
@@ -11603,6 +11645,11 @@ mod tests {
] {
rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
}
if minio_unversioned {
object_meta
.meta_sys
.insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
}
*shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
.expect("legacy transitioned version should re-encode");
}
@@ -11613,6 +11660,152 @@ mod tests {
}
}
#[cfg(feature = "test-util")]
async fn read_store_body(
store: &Arc<crate::store::ECStore>,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
opts: &ObjectOptions,
) -> Vec<u8> {
let mut reader = store
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
.await
.expect("object reader should open");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("object body should drain");
body
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
backend.set_put_remote_version(Some(String::new())).await;
let bucket = "legacy-unknown-unversioned-read-bucket";
let object = "object.bin";
let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("legacy source bucket should be created");
let mut reader = PutObjReader::from_vec(payload.clone());
let source = store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("legacy source should be written");
store
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.to_string(),
etag: source.etag.clone().expect("legacy source should have an etag"),
..Default::default()
},
mod_time: source.mod_time,
..Default::default()
},
)
.await
.expect("legacy source should transition");
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
backend.clear_op_log().await;
let opts = ObjectOptions {
metadata_cache_safe: false,
..Default::default()
};
let head = store
.get_object_info(bucket, object, &opts)
.await
.expect("legacy transitioned HEAD should use local metadata");
assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
assert!(head.transitioned_object.version_id.is_empty());
assert_eq!(
head.user_defined
.get("x-minio-internal-transitioned-versionID")
.map(String::as_str),
Some(""),
"the MinIO empty version-key provenance must survive xl.meta decoding"
);
assert!(
!rustfs_utils::http::metadata_compat::contains_key_str(
&head.user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
),
"the compatibility read must not synthesize version-state metadata"
);
let full_body = read_store_body(&store, bucket, object, None, &opts).await;
assert_eq!(full_body, payload);
let range = HTTPRangeSpec {
is_suffix_length: false,
start: 7,
end: 38,
};
let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
assert_eq!(ranged_body, &payload[7..=38]);
let after_read = store.pools[0]
.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
.await
.expect("legacy metadata should remain readable after GET")
.expect("legacy object metadata should remain on disk")
.versions
.into_iter()
.find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
.expect("legacy transitioned source should remain visible after GET");
assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
assert!(after_read.transition_version.is_none());
assert!(after_read.transition_version_id.is_none());
assert_eq!(
after_read
.metadata
.get("x-minio-internal-transitioned-versionID")
.map(String::as_str),
Some(""),
"the MinIO empty version-key provenance must remain after GET and Range GET"
);
assert!(
!rustfs_utils::http::metadata_compat::contains_key_str(
&after_read.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
),
"the compatibility read must remain side-effect free"
);
assert_eq!(
backend.op_log().await,
vec![
MockWarmOp::Probe {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Get {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Probe {
object: after_read.transitioned_objname.clone(),
},
MockWarmOp::Get {
object: after_read.transitioned_objname,
},
],
"legacy reads should probe before each unversioned GET and never mutate local metadata"
);
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -11653,7 +11846,7 @@ mod tests {
)
.await
.expect("legacy source should transition");
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await;
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
let legacy = store.pools[0]
.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
@@ -12794,7 +12987,7 @@ mod tests {
.expect("merge-loser source should transition");
copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
}
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await;
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
backend.set_remove_failure(true);
store.pools[1]
.delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
@@ -16861,6 +17054,10 @@ mod tests {
.find(|version| version.version_id == history.version_id)
.expect("transitioned history should exist");
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
rustfs_utils::http::metadata_compat::remove_str(
&mut transitioned.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
);
metadata
.add_version(transitioned)
.expect("unknown state should replace the transitioned version");
@@ -17196,6 +17393,147 @@ mod tests {
assert!(matches!(err, Error::ConfigNotFound));
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
let probe_id = uuid::Uuid::new_v4();
let creator_epoch = uuid::Uuid::new_v4();
let initial = TierProbeIntent {
probe_id,
revision: 1,
state: TierProbeIntentState::UploadOutcomeUnknown,
operation: TierProbeOperationIdentity::Verify {
config_etag: "config-etag".to_string(),
backend_identity: [1; 32],
},
tier_name: "COLD-A".to_string(),
destination_id: [1; 32],
probe_object: format!("rustfs-tier-probe-{probe_id}"),
creator_id: "node-a".to_string(),
creator_epoch,
created_at_unix_nanos: 1_780_000_000_000_000_000,
owner: TierProbeOwnerFence {
owner_id: "node-a".to_string(),
owner_epoch: creator_epoch,
not_after_unix_nanos: 1_780_000_900_000_000_000,
},
remote_version: TierProbeRemoteVersion::default(),
};
save_tier_probe_intent_record_if_absent(store.clone(), &initial)
.await
.expect("initial probe intent should persist with create-only semantics");
let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
.await
.expect_err("duplicate create must fail closed");
assert!(matches!(duplicate, Error::PreconditionFailed));
let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("initial probe intent should load with an ETag");
assert_eq!(observed_initial.intent(), &initial);
let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
.await
.expect_err("nonterminal evidence must not be deleted");
assert!(nonterminal_delete.to_string().contains("must be terminal"));
let mut fabricated_current_intent = initial.clone();
fabricated_current_intent.tier_name = "COLD-B".to_string();
let mut fabricated_successor = fabricated_current_intent.clone();
fabricated_successor
.advance(
TierProbeIntentState::Uploaded,
TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
)
.expect("fabricated successor should be internally valid");
let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
.await
.expect_err("a live ETag must not authorize a different caller record");
assert!(matches!(crossed_cas, Error::PreconditionFailed));
assert_eq!(
load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("crossed CAS must retain the authoritative record")
.intent(),
&initial
);
let mut fabricated_terminal_intent = fabricated_current_intent;
fabricated_terminal_intent
.advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
.expect("fabricated terminal should be internally valid");
let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
.await
.expect_err("a live ETag must not delete for a different caller record");
assert!(matches!(crossed_delete, Error::PreconditionFailed));
assert_eq!(
load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("crossed delete must retain the authoritative record")
.intent(),
&initial
);
let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
let mut uploaded = observed_initial.intent().clone();
uploaded
.advance(TierProbeIntentState::Uploaded, remote_version.clone())
.expect("known PUT result should advance");
save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
.await
.expect("the matching initial ETag should admit one successor");
let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
.await
.expect_err("a consumed ETag must not overwrite the current generation");
assert!(matches!(stale_cas, Error::PreconditionFailed));
let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("uploaded generation should load");
assert_eq!(observed_uploaded.intent(), &uploaded);
let mut cleanup = observed_uploaded.intent().clone();
cleanup
.advance(TierProbeIntentState::CleanupPending, remote_version.clone())
.expect("known candidate should become cleanup-pending");
save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
.await
.expect("cleanup generation should persist by exact ETag");
let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("cleanup generation should load");
let mut completed = observed_cleanup.intent().clone();
completed
.advance(TierProbeIntentState::Completed, remote_version)
.expect("exact cleanup should become terminal");
save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
.await
.expect("terminal generation should persist by exact ETag");
let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
.await
.expect_err("a stale ETag must not delete terminal evidence");
assert!(matches!(stale_delete, Error::PreconditionFailed));
let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
.await
.expect("terminal generation should remain after stale delete");
assert_eq!(observed_completed.intent(), &completed);
delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
.await
.expect("the exact terminal ETag should delete the record");
assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
crates/ecstore/src/store/mod.rs
+1 -1
View File
@@ -425,7 +425,7 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pub use object::DeleteAfterObjectLockSnapshotBarrier;
pub(crate) use object::{
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
crates/filemeta/src/filemeta/version.rs
+115 -10
View File
@@ -297,6 +297,20 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio
}
}
fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String {
decoded.map(str::to_owned).unwrap_or_else(|| {
if raw.is_empty() {
String::new()
} else {
String::from_utf8_lossy(raw).into_owned()
}
})
}
fn is_transition_version_metadata_key(key: &str) -> bool {
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
}
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
let valid = match state {
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
@@ -366,14 +380,26 @@ impl<'a> DerivedInternalMetadata<'a> {
}
*slot = Some(value.as_slice());
}
fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result<Option<&'a [u8]>> {
if let (Some(canonical), Some(legacy)) = (canonical, legacy)
&& canonical != legacy
{
return Err(Error::FileCorrupt);
}
Ok(canonical.or(legacy))
}
Ok(Self {
checksum: canonical.checksum.or(legacy.checksum),
part_checksums: canonical.part_checksums.or(legacy.part_checksums),
transition_status: canonical.transition_status.or(legacy.transition_status),
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
transition_tier: canonical.transition_tier.or(legacy.transition_tier),
transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?,
transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?,
transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?,
transitioned_version_state: merge_consistent(
canonical.transitioned_version_state,
legacy.transitioned_version_state,
)?,
transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?,
})
}
}
@@ -438,8 +464,14 @@ impl FileInfo {
}
}
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
if state == TransitionVersionState::Unknown {
fn set_transition_version_state(
meta_sys: &mut HashMap<String, Vec<u8>>,
state: TransitionVersionState,
source_metadata: &HashMap<String, String>,
) {
if state == TransitionVersionState::Unknown
&& !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE)
{
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
} else {
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
@@ -2643,6 +2675,11 @@ impl MetaObject {
if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
}
for (key, value) in &self.meta_sys {
if is_transition_version_metadata_key(key) {
metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref()));
}
}
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
let transition_tier = derived_metadata
.transition_tier
@@ -2689,7 +2726,7 @@ impl MetaObject {
} else {
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
}
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata);
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
@@ -2830,7 +2867,7 @@ impl From<FileInfo> for MetaObject {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_status.is_empty() {
set_transition_version_state(&mut meta_sys, value.transition_version_state);
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
}
if !value.transition_tier.is_empty() {
@@ -2985,6 +3022,12 @@ impl MetaDeleteMarker {
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
fi.transition_version =
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
for (key, value) in &self.meta_sys {
if is_transition_version_metadata_key(key) {
fi.metadata
.insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref()));
}
}
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
@@ -3152,7 +3195,7 @@ impl From<FileInfo> for MetaDeleteMarker {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_status.is_empty() || value.tier_free_version() {
set_transition_version_state(&mut meta_sys, value.transition_version_state);
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
}
if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
@@ -4574,6 +4617,7 @@ mod tests {
.into_fileinfo("b", "k", false)
.expect("into_fileinfo");
assert_eq!(fi.transition_version_id, None);
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new()));
}
#[test]
@@ -4585,6 +4629,10 @@ mod tests {
.into_fileinfo("b", "k", false)
.expect("into_fileinfo");
assert_eq!(fi.transition_version_id, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"nil UUID bytes must remain distinguishable from an empty MinIO version"
);
}
#[test]
@@ -4598,6 +4646,7 @@ mod tests {
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
}
#[test]
@@ -4637,6 +4686,36 @@ mod tests {
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
}
#[test]
fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
SUFFIX_TRANSITIONED_VERSION_STATE,
TransitionVersionState::Unknown.as_str().to_string(),
);
let fi = FileInfo {
transition_status: "complete".to_string(),
transition_version_state: TransitionVersionState::Unknown,
metadata,
..Default::default()
};
let object = MetaObject::from(fi);
assert_eq!(
get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE),
Some(b"unknown".as_slice())
);
let decoded = object
.into_fileinfo("b", "k", false)
.expect("explicit unknown state should decode");
assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown);
assert_eq!(
rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,),
Some("unknown")
);
}
#[test]
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
let id = sample_version_id();
@@ -4753,6 +4832,10 @@ mod tests {
.expect("invalid transition version bytes must not fail the object read");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"invalid raw bytes must remain distinguishable from an empty MinIO version"
);
}
#[test]
@@ -4795,6 +4878,10 @@ mod tests {
.into_fileinfo("b", "k", false)
.expect("nil tier version should remain an absent remote version");
assert_eq!(fi.transition_version_id, None);
assert!(
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
"nil UUID bytes must remain distinguishable from an empty MinIO version"
);
}
#[test]
@@ -4812,6 +4899,7 @@ mod tests {
.expect("legacy binary UUID tier version should decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
}
#[test]
@@ -4910,6 +4998,23 @@ mod tests {
assert_eq!(err, Error::FileCorrupt);
}
#[test]
fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() {
let sys = HashMap::from([
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"),
b"unknown".to_vec(),
),
("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()),
]);
let err = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect_err("mixed-case transition state aliases must agree");
assert_eq!(err, Error::FileCorrupt);
}
#[test]
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
let object = FileMetaVersionHeader {
crates/scanner/src/remote_scanner.rs
+3
View File
@@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
mod stream;
#[cfg(test)]
pub(crate) use stream::checkpoint_fixture_partial_return;
pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};
crates/scanner/src/remote_scanner/stream.rs
+42
View File
@@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream(
#[cfg(test)]
const TEST_NEXT_CYCLE: u64 = 11;
#[cfg(test)]
pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
let request_id = Uuid::new_v4();
let writer_auth = FrameAuthenticator::for_test(request_id);
let reader_auth = FrameAuthenticator::for_test(request_id);
let mut bytes = Vec::new();
write_frame(
&mut bytes,
&writer_auth,
&mut 0,
&RemoteScannerFrame::terminal(
RemoteScannerProgress {
objects_scanned: progress.0,
directories_started: progress.1,
entries_visited,
},
RemoteScannerFrameResult::Partial,
),
)
.await
.expect("checkpoint partial frame must encode");
let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
.await
.expect("checkpoint progress frame must authenticate");
assert_eq!(frame.progress.entries_visited, entries_visited);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
let result = consume_remote_scanner_stream(
std::io::Cursor::new(bytes),
parent,
budget.clone(),
"bucket",
DataUsageCacheSource::new(0, 0),
DataUsageScanPlanDigest([17; 32]),
reader_auth,
)
.await
.expect("checkpoint partial frame must decode");
assert!(matches!(result, RemoteScannerOutcome::Partial));
assert_eq!(budget.progress(), progress);
}
#[cfg(test)]
async fn consume_remote_scanner_stream<R>(
reader: R,
crates/scanner/src/scanner_folder/tests.rs
+2
View File
@@ -24,6 +24,8 @@ use std::io::Write;
use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::Mutex;
mod checkpoint_fixture;
/// Reset the process-global alert cooldown map; test-only.
fn reset_alert_cooldowns() {
*SCANNER_ALERT_EMISSION_COOLDOWN
crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs
+410
View File
@@ -0,0 +1,410 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
use crate::storage_api::scanner_io::ObjectIO;
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
const STATIC_OBJECTS: u64 = 24;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
#[derive(Debug)]
struct FixtureStore {
root: tempfile::TempDir,
reject_save: AtomicBool,
}
impl FixtureStore {
fn new() -> Arc<Self> {
Arc::new(Self {
root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
reject_save: AtomicBool::new(false),
})
}
fn path(&self, object: &str) -> std::path::PathBuf {
assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
self.root
.path()
.join(if object.ends_with(".bkp") { "backup" } else { "main" })
}
async fn strict_load(&self) -> DataUsageCache {
let bytes = tokio::fs::read(self.root.path().join("main"))
.await
.expect("saved checkpoint fixture must exist");
decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
}
}
#[async_trait::async_trait]
impl ObjectIO for FixtureStore {
type Error = crate::EcstoreError;
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
type HeaderMap = http::HeaderMap;
type ObjectOptions = crate::ScannerObjectOptions;
type ObjectInfo = crate::ScannerObjectInfo;
type GetObjectReader = crate::ScannerGetObjectReader;
type PutObjectReader = crate::ScannerPutObjReader;
async fn get_object_reader(
&self,
_bucket: &str,
object: &str,
_range: Option<Self::RangeSpec>,
_headers: Self::HeaderMap,
_options: &Self::ObjectOptions,
) -> crate::EcstoreResult<Self::GetObjectReader> {
let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
crate::EcstoreError::FileNotFound
} else {
crate::EcstoreError::from(error)
}
})?;
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
Ok(crate::ScannerGetObjectReader {
stream: Box::new(Cursor::new(bytes)),
object_info: crate::ScannerObjectInfo {
etag: Some("fixture".into()),
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
})
}
async fn put_object(
&self,
_bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
options: &Self::ObjectOptions,
) -> crate::EcstoreResult<Self::ObjectInfo> {
if self.reject_save.load(Ordering::SeqCst) {
return Err(crate::EcstoreError::PreconditionFailed);
}
let path = self.path(object);
let exists = tokio::fs::try_exists(&path).await?;
let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
if (exists && preconditions.if_none_match_value() == Some("*"))
|| (!exists && preconditions.if_match_value().is_some())
|| (exists && preconditions.if_match_value() != Some("fixture"))
{
return Err(crate::EcstoreError::PreconditionFailed);
}
let mut bytes = Vec::new();
(&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
tokio::fs::write(path, bytes).await?;
Ok(crate::ScannerObjectInfo {
etag: Some("fixture".into()),
..Default::default()
})
}
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for FixtureStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_options: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
return Err("missing or oversized checkpoint fixture");
}
let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
return Err("checkpoint fixture has no valid bucket root");
}
Ok(cache)
}
fn retained(cache: &DataUsageCache) -> u64 {
assert!(
!cache.root().is_some_and(|root| root.compacted),
"a compacted bucket root cannot prove static-prefix coverage"
);
cache
.checked_flatten("bucket/static")
.map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
}
#[derive(Debug, PartialEq, Eq)]
enum CoverageDiagnosis {
Progress,
NoNewWork,
LostAtPrepare,
LostAtReload,
WalkWithoutRetention,
}
fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
if reloaded < scanned {
CoverageDiagnosis::LostAtReload
} else if prepared < previous {
CoverageDiagnosis::LostAtPrepare
} else if walked > 0 && reloaded <= previous {
CoverageDiagnosis::WalkWithoutRetention
} else if reloaded > previous {
CoverageDiagnosis::Progress
} else {
CoverageDiagnosis::NoNewWork
}
}
#[test]
fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
}
#[test]
fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
for bytes in [
vec![],
vec![0xc1],
DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
] {
assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
}
}
#[test]
fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
let mut cache = DataUsageCache::default();
cache.info.name = "bucket".to_string();
cache.replace("bucket", "", DataUsageEntry::default());
cache.replace("bucket/static", "bucket", DataUsageEntry::default());
for index in 0..4 {
cache.replace(
&format!("bucket/static/{index}"),
"bucket/static",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
}
cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
let entry = decoded
.find("bucket/static")
.expect("compaction must retain the static subtree root");
assert!(entry.compacted);
assert!(entry.children.is_empty());
assert_eq!(
retained(&decoded),
4,
"compaction retains aggregate coverage even when leaf keys are absent"
);
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_save_reload_resume() {
run_checkpoint_fixture(false).await;
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_hot_digest_diagnostic() {
run_checkpoint_fixture(true).await;
}
async fn run_checkpoint_fixture(change_digest: bool) {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for index in 0..STATIC_OBJECTS {
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
}
let store = FixtureStore::new();
let mut previous = 0;
let mut visited = 0;
for round in 0..3_u8 {
write_test_object_metadata(&root, "bucket", "hot/current").await;
let mut cache = DataUsageCache::default();
let revisions = cache
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("load checkpoint revisions");
if round > 0 {
assert_eq!(retained(&store.strict_load().await), previous);
}
let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut cache,
"bucket",
SOURCE,
11,
7,
plan,
crate::scanner_io::DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: None,
},
);
let prepared = retained(&cache);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(4),
..Default::default()
},
);
let outcome = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("budgeted local disk scan returns partial cache");
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
panic!("budgeted fixture must remain partial")
};
assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
let scanned = retained(&cache);
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("persist partial checkpoint");
let mut loaded = DataUsageCache::default();
loaded
.load(store.clone(), CACHE_NAME)
.await
.expect("reload persisted partial checkpoint");
let reloaded = retained(&loaded);
assert_eq!(reloaded, retained(&store.strict_load().await));
assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
assert!(!loaded.info.snapshot_complete);
visited += budget.entries_visited();
let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
eprintln!(
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
);
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
assert_eq!(
diagnosis,
CoverageDiagnosis::Progress,
"visited growth must produce durable static coverage"
);
}
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
previous = reloaded;
}
assert!(visited > 0, "fixture must exercise the directory walk");
assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
let mut loaded = DataUsageCache::default();
let revisions = loaded
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("load final checkpoint");
let before = tokio::fs::read(store.root.path().join("main"))
.await
.expect("read durable checkpoint bytes");
let epoch_error = loaded
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
.await
.expect_err("stale publication epoch must reject persistence");
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
store.reject_save.store(true, Ordering::SeqCst);
loaded.info.next_cycle += 1;
loaded
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect_err("injected save failure must not report durable progress");
assert_eq!(
tokio::fs::read(store.root.path().join("main"))
.await
.expect("read unchanged checkpoint bytes"),
before
);
let parent = CancellationToken::new();
parent.cancel();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
loaded.clone(),
None,
HealScanMode::Normal,
)
.await;
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
loaded,
None,
HealScanMode::Normal,
)
.await
.expect("unbounded scan must complete after durable partial progress");
let ScannerDiskScanOutcome::Complete(cache) = result else {
panic!("unbounded fixture must produce a complete disk cache");
};
assert!(cache.info.snapshot_complete);
assert!(cache.info.scan_checkpoint.is_none());
assert_eq!(
cache.checked_flatten("bucket").expect("complete bucket root").objects,
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
);
}
crates/scanner/src/scanner_io.rs
+17
View File
@@ -105,6 +105,12 @@ struct DirtyUsageSnapshot {
covers_all_pending: bool,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct ScannerBucketScanScope {
selected_buckets: Option<Arc<HashSet<String>>>,
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
}
@@ -146,6 +152,7 @@ fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
pub struct ScannerBucketScanPlan {
buckets: Vec<BucketInfo>,
all_buckets: Arc<Vec<BucketInfo>>,
scope: ScannerBucketScanScope,
digest: DataUsageScanPlanDigest,
leader_epoch: u64,
tier_registry_generation: u64,
@@ -202,6 +209,14 @@ fn scanner_bucket_cache_digest(
DataUsageScanPlanDigest(hasher.finalize().into())
}
#[cfg(test)]
pub(crate) fn checkpoint_fixture_bucket_digest(
scan_plan_digest: DataUsageScanPlanDigest,
dirty_generation: Option<u64>,
) -> DataUsageScanPlanDigest {
scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
}
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
if results.iter().any(|result| result.info.last_update.is_some()) {
return Ok(());
@@ -732,6 +747,8 @@ mod dirty_usage;
mod guards;
mod io_cache;
mod io_cycle;
#[cfg(test)]
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
pub(crate) use io_cycle::nsscanner_with_storage_status;
mod io_disk;
#[cfg(test)]
crates/scanner/src/scanner_io/io_cache.rs
+230 -106
View File
@@ -14,6 +14,93 @@
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use super::*;
#[derive(Clone, Copy)]
pub(super) struct ScannerSetCacheGeneration {
pub(super) want_cycle: u64,
pub(super) leader_epoch: u64,
pub(super) tier_registry_generation: u64,
pub(super) source: DataUsageCacheSource,
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
}
pub(super) struct PreparedScopedSetScan {
pub(super) buckets: Vec<BucketInfo>,
pub(super) cache: DataUsageCache,
}
pub(super) fn prepare_scoped_set_scan(
old_cache: &DataUsageCache,
set_buckets: &[BucketInfo],
all_buckets: &[BucketInfo],
scope: &ScannerBucketScanScope,
generation: ScannerSetCacheGeneration,
) -> Option<PreparedScopedSetScan> {
let (Some(selected_buckets), Some(baseline_scan_plan_digest)) = (&scope.selected_buckets, scope.baseline_scan_plan_digest)
else {
return None;
};
if selected_buckets.is_empty()
|| !old_cache.info.snapshot_complete
|| old_cache.info.last_update.is_none()
|| old_cache.info.name != DATA_USAGE_ROOT
|| old_cache.info.next_cycle > generation.want_cycle
|| old_cache.info.leader_epoch != generation.leader_epoch
|| old_cache.info.tier_registry_generation != Some(generation.tier_registry_generation)
|| old_cache.info.source != Some(generation.source)
|| old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest)
|| old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
|| old_cache.checked_flatten_complete_scope(DATA_USAGE_ROOT).is_none()
{
return None;
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: generation.want_cycle,
leader_epoch: generation.leader_epoch,
tier_registry_generation: Some(generation.tier_registry_generation),
source: Some(generation.source),
snapshot_complete: false,
scan_plan_digest: Some(generation.scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
lkg_snapshot_complete: true,
lkg_next_cycle: Some(old_cache.info.next_cycle),
lkg_last_update: old_cache.info.last_update,
lkg_leader_epoch: Some(old_cache.info.leader_epoch),
lkg_scan_plan_digest: old_cache.info.scan_plan_digest,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let root_hash = crate::hash_path(DATA_USAGE_ROOT);
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
}
if selected_buckets.contains(&bucket.name) {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
continue;
}
let bucket_hash = crate::hash_path(&bucket.name);
old_cache.find(&bucket.name)?;
cache.copy_with_children(old_cache, &bucket_hash, &Some(root_hash.clone()));
cache.find(&bucket.name)?;
}
Some(PreparedScopedSetScan {
buckets: set_buckets
.iter()
.filter(|bucket| selected_buckets.contains(&bucket.name))
.cloned()
.collect(),
cache,
})
}
#[async_trait::async_trait]
impl ScannerIOCache for SetDisks {
#[tracing::instrument(skip(self, budget, scan_plan, updates))]
@@ -27,8 +114,9 @@ impl ScannerIOCache for SetDisks {
scan_mode: HealScanMode,
) -> Result<()> {
let ScannerBucketScanPlan {
buckets,
mut buckets,
all_buckets,
scope,
digest: scan_plan_digest,
leader_epoch,
tier_registry_generation,
@@ -63,26 +151,57 @@ impl ScannerIOCache for SetDisks {
"Scanner old data usage cache load failed; rebuilding from bucket caches"
);
}
let scoped_scan = prepare_scoped_set_scan(
&old_cache,
&buckets,
&all_buckets,
&scope,
ScannerSetCacheGeneration {
want_cycle,
leader_epoch,
tier_registry_generation,
source,
scan_plan_digest,
},
);
let mut scoped_cache = scoped_scan.map(|prepared| {
buckets = prepared.buckets;
prepared.cache
});
if buckets.is_empty() {
let now = SystemTime::now();
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
last_update: Some(now),
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
let mut cache = match scoped_cache.take() {
Some(cache) => cache,
None => {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
cache
}
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
cache.info.last_update = Some(now);
cache.info.snapshot_complete = true;
cache.info.lkg_snapshot_complete = false;
cache.info.lkg_next_cycle = None;
cache.info.lkg_last_update = None;
cache.info.lkg_leader_epoch = None;
cache.info.lkg_scan_plan_digest = None;
if cache.find(DATA_USAGE_ROOT).is_none() {
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(
@@ -269,92 +388,102 @@ impl ScannerIOCache for SetDisks {
record_disk_bucket_scans_active(0, &pool_label, &set_label);
let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());
// Fence a stale set aggregate before copying entries into per-bucket work caches.
if old_cache.info.next_cycle <= want_cycle
&& old_cache.info.leader_epoch <= leader_epoch
&& old_cache.info.tier_registry_generation != Some(tier_registry_generation)
{
old_cache.info.scan_plan_digest = None;
}
let old_lkg = old_cache.info.snapshot_complete.then_some({
(
old_cache.info.next_cycle,
old_cache.info.last_update,
old_cache.info.leader_epoch,
old_cache.info.scan_plan_digest,
)
});
let prepare_outcome = match old_cache.prepare_for_scan(
DATA_USAGE_ROOT,
want_cycle,
leader_epoch,
source,
scan_plan_digest,
require_cache_source,
) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = want_cycle,
cached_cycle = old_cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return Ok(());
let mut cache = if let Some(cache) = scoped_cache.take() {
cache
} else {
// Fence a stale set aggregate before copying entries into per-bucket work caches.
if old_cache.info.next_cycle <= want_cycle
&& old_cache.info.leader_epoch <= leader_epoch
&& old_cache.info.tier_registry_generation != Some(tier_registry_generation)
{
old_cache.info.scan_plan_digest = None;
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = leader_epoch,
cached_epoch = old_cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected work from an older leader epoch"
);
return Ok(());
}
outcome => outcome,
};
if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
&& let Some((cycle, last_update, epoch, digest)) = old_lkg
{
old_cache.info.lkg_snapshot_complete = true;
old_cache.info.lkg_next_cycle = Some(cycle);
old_cache.info.lkg_last_update = last_update;
old_cache.info.lkg_leader_epoch = Some(epoch);
old_cache.info.lkg_scan_plan_digest = digest;
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
let old_lkg = old_cache.info.snapshot_complete.then_some({
(
old_cache.info.next_cycle,
old_cache.info.last_update,
old_cache.info.leader_epoch,
old_cache.info.scan_plan_digest,
)
});
let prepare_outcome = match old_cache.prepare_for_scan(
DATA_USAGE_ROOT,
want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
source,
scan_plan_digest,
require_cache_source,
) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = want_cycle,
cached_cycle = old_cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return Ok(());
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = leader_epoch,
cached_epoch = old_cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected work from an older leader epoch"
);
return Ok(());
}
outcome => outcome,
};
if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
&& let Some((cycle, last_update, epoch, digest)) = old_lkg
{
old_cache.info.lkg_snapshot_complete = true;
old_cache.info.lkg_next_cycle = Some(cycle);
old_cache.info.lkg_last_update = last_update;
old_cache.info.lkg_leader_epoch = Some(epoch);
old_cache.info.lkg_scan_plan_digest = digest;
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
lkg_snapshot_complete: old_cache.info.lkg_snapshot_complete,
lkg_next_cycle: old_cache.info.lkg_next_cycle,
lkg_last_update: old_cache.info.lkg_last_update,
lkg_leader_epoch: old_cache.info.lkg_leader_epoch,
lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
cache
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
let (bucket_tx, bucket_rx) = mpsc::channel::<BucketInfo>(buckets.len());
@@ -1257,11 +1386,6 @@ impl ScannerIOCache for SetDisks {
incomplete_scope.info.snapshot_complete = false;
incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete;
incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle;
incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update;
incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch;
incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest;
if let Err(e) = updates.send(incomplete_scope).await {
error!(
target: "rustfs::scanner::io",
crates/scanner/src/scanner_io/io_cycle.rs
+36
View File
@@ -63,6 +63,41 @@ pub(crate) async fn nsscanner_with_storage_status<S>(
where
S: ScannerStorage,
{
let request = ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle,
leader_epoch,
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
};
nsscanner_with_storage_status_scoped(store, request).await
}
pub(crate) struct ScannerCycleRequest {
pub(crate) ctx: CancellationToken,
pub(crate) budget: Arc<ScannerCycleBudget>,
pub(crate) updates: mpsc::Sender<DataUsageInfo>,
pub(crate) want_cycle: u64,
pub(crate) leader_epoch: u64,
pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope,
}
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
where
S: ScannerStorage,
{
let ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle,
leader_epoch,
scan_mode,
scan_scope,
} = request;
let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -280,6 +315,7 @@ where
let scan_plan = ScannerBucketScanPlan {
buckets: set_buckets,
all_buckets: Arc::clone(&all_buckets),
scope: scan_scope.clone(),
digest: scan_plan_digest,
leader_epoch,
tier_registry_generation,
crates/scanner/src/scanner_io/tests.rs
+170
View File
@@ -765,6 +765,155 @@ fn bucket_usage_scan_order_prioritizes_dirty_buckets() {
assert_eq!(names, vec!["dirty", "missing", "cached"]);
}
fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsageScanPlanDigest) -> DataUsageCache {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: 7,
last_update: Some(SystemTime::now()),
leader_epoch: 11,
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
tier_registry_generation: Some(13),
..Default::default()
},
..Default::default()
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for (bucket, size) in buckets {
cache.replace(
bucket,
DATA_USAGE_ROOT,
DataUsageEntry {
size: *size,
objects: 1,
..Default::default()
},
);
}
cache
}
#[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let current_digest = DataUsageScanPlanDigest([2; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20), ("deleted", 30)], baseline_digest);
old_cache.replace(
"stable/prefix",
"stable",
DataUsageEntry {
size: 5,
objects: 1,
..Default::default()
},
);
let all_buckets = vec![bucket_info("stable"), bucket_info("dirty")];
let selected_buckets = Arc::new(HashSet::from(["dirty".to_string(), "deleted".to_string()]));
let prepared = prepare_scoped_set_scan(
&old_cache,
&all_buckets,
&all_buckets,
&ScannerBucketScanScope {
selected_buckets: Some(selected_buckets),
baseline_scan_plan_digest: Some(baseline_digest),
},
ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: current_digest,
},
)
.expect("complete matching set cache should support a scoped scan");
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
let stable = prepared
.cache
.checked_flatten("stable")
.expect("unselected bucket subtree should be retained");
assert_eq!((stable.size, stable.objects), (15, 2));
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
assert_eq!(prepared.cache.info.scan_plan_digest, Some(current_digest));
assert_eq!(prepared.cache.info.next_cycle, 8);
assert!(!prepared.cache.info.snapshot_complete);
assert!(prepared.cache.info.lkg_snapshot_complete);
assert_eq!(prepared.cache.info.lkg_next_cycle, Some(7));
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
let baseline_digest = DataUsageScanPlanDigest([3; 32]);
let old_cache = complete_set_usage_cache(&[("stable", 10)], baseline_digest);
let all_buckets = vec![bucket_info("stable"), bucket_info("new")];
assert!(
prepare_scoped_set_scan(
&old_cache,
&all_buckets,
&all_buckets,
&ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
},
ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([4; 32]),
},
)
.is_none()
);
}
#[test]
fn scoped_set_scan_requires_an_exact_complete_baseline() {
let baseline_digest = DataUsageScanPlanDigest([5; 32]);
let all_buckets = vec![bucket_info("dirty")];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([6; 32]),
};
let mut incomplete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
incomplete.info.snapshot_complete = false;
assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut not_durable = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
not_durable.info.last_update = None;
assert!(prepare_scoped_set_scan(&not_durable, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32]));
assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none());
let empty_scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::new())),
baseline_scan_plan_digest: Some(baseline_digest),
};
let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none());
let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
future_cache.info.next_cycle = generation.want_cycle.saturating_add(1);
assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation).is_none());
}
#[test]
fn record_set_scan_failure_preserves_first_error() {
let mut first = None;
@@ -899,6 +1048,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
}
}
#[test]
fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
for (budget, cancelled, bucket, expected) in [
(false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
(true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
(false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
] {
assert_eq!(
classify_nsscanner_cycle(
true,
budget,
cancelled,
bucket,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unchanged
),
expected,
);
}
}
#[test]
fn unverified_activity_defers_partial_and_floor_cycles() {
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
docs/architecture/README.md
+1 -1
View File
@@ -48,7 +48,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format and its mixed-version rules |
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |
| [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request |
## Support and compatibility matrices (release-facing, keep current)
docs/architecture/heal-concurrency-model.md
+1 -1
View File
@@ -7,7 +7,7 @@ For crate ownership, read [crate-boundaries.md](crate-boundaries.md): ECStore ow
## Model
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. This describes the intended lock scope while the guard remains valid; it does not prove rejection of an already-dispatched disk syscall after distributed lease loss. The authority, delayed-mutation, and recovery boundary is specified in [unified-object-generation.md](unified-object-generation.md). MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
| Layer | Mechanism | Owner |
| --- | --- | --- |
docs/architecture/ilm-tiering-persistence-contracts.md
+206 -30
View File
@@ -1,7 +1,7 @@
# ILM And Tiering Persistence Contracts
**Use this when:** changing an ILM transition, tier configuration mutation, manual transition job, tier-delete recovery path, pool decommission, or any code that can create, transfer, or destroy ownership of a remote-tier object.
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; the dormant validation-probe record and conditional primitives in `crates/ecstore/src/services/tier/tier_probe_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
This document separates three kinds of statement:
@@ -41,9 +41,10 @@ All keys below are objects in the internal metadata bucket. The table gives the
| Protocol | Current schema/version | Canonical key | Creator and cleanup owner | Authoritative identity and mutable fields | Current durability point |
|---|---|---|---|---|---|
| Transition transaction | `rustfs-transition-transaction-v1` | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Maximum-parity config write. Current create/update/delete calls do not use ETag preconditions |
| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete |
| Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records/<aa>/<bb>/<mutation-id>.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity |
| Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators/<aa>/<bb>/<mutation-id>.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent |
| Tier validation probe intent | Dormant `rustfs-tier-probe-intent-v1`; no writer or recovery is enabled | `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json` | No current runtime owner because no path creates the record; v1 permits only the immutable creator as owner | Immutable probe, operation-generation, destination, random remote object, creator identity, and v1 owner fence. Mutable: revision, state, and monotonic remote-version proof | Conditional create/CAS/delete primitives exist but are not called by Add/Edit/Verify or recovery |
| Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs/<aa>/<bb>/<job-id>.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS |
| Manual scope admission | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/scopes/<aa>/<bb>/<scope-digest>.json` | Job admission creates/renews it; the job owner removes it after terminalization | Immutable bucket/run-vs-dry-run scope; mutable job/lease ownership and expiry | Create-only, renew/delete by ETag CAS |
| Manual task | `rustfs-manual-transition-task-v1` | `ilm/manual-transition/tasks/<job shards>/<job-id>/<task-key>.json` | The scanner persists it before queue admission; no current GC owner | Immutable job plus exact bucket/object/version/tier work identity | Append-only create with `If-None-Match: *` and maximum parity |
@@ -54,8 +55,9 @@ All keys below are objects in the internal metadata bucket. The table gives the
| Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/<scope-digest>.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion |
| Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts/<run-token>/<source-path>/<id-kind>/<id>.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity |
| Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup |
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transition-transaction namespace, the dormant tier-validation-probe namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
## Durable fences and write primitives
@@ -71,7 +73,7 @@ The internal config layer supplies maximum-parity writes, create-only writes, ET
| Conditional delete | Removes only the verified terminal generation; if an active decommission covers it, its terminal receipt is written first |
| Strong readback | Resolves a lost response only when key, schema, full immutable identity, state, and expected successor all match |
Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. The transition transaction currently carries a fixed `owner_epoch` fence identity and a mutable `revision`, but persists with unconditional writes and deletes; those fields therefore detect some in-memory misuse but are not yet a durable exclusion fence. Changing `owner_epoch` during takeover is not current behavior and remains an open design. The manual job's initial UUID-key write has the same create-only gap, although all later owner/lease updates are CAS-protected.
Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. Transition transaction v1 now uses create-only installation, exact record plus ETag read before each successor CAS, and exact ETag terminal deletion. Its `owner_epoch` and `not_after_unix_nanos` remain immutable, however, so an expired recovery worker claims only the next state generation rather than a renewable durable owner lease. The manual job's initial UUID-key write still lacks create-only installation, although all later owner/lease updates are CAS-protected.
### Approved target
@@ -81,7 +83,7 @@ Tier mutation intents and v6 journal/manifest records use the conditional primit
- After a timeout, connection loss, or quorum-uncertain response, the caller must strongly reread. Only the exact intended successor is success; predecessor, absence, conflict, corruption, or unavailable readback retains the record and blocks destructive action.
- A process-local mutex, cancellation token, task registry, or cached generation may reduce duplicate work but cannot authorize publication, rollback, or remote deletion.
The exact transition-transaction lease/takeover fields and whether the existing `not_after` becomes the owner expiry are an **open design**. They must be settled with upgrade/downgrade behavior before the v1 schema changes.
The approved transition-transaction successor, lease/takeover fields, v1 migration, and upgrade/downgrade gates are specified in [Bounded recovery control and operator disposition](#bounded-recovery-control-and-operator-disposition). They require implementation and fleet gating before any v2 writer or destructive v1 takeover is enabled.
## Lock and operation order
@@ -89,18 +91,19 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n
| Path | Current acquisition order | Operations allowed while held | Operations forbidden while held |
|---|---|---|---|
| Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation |
| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. Both guards are released for lease drain, peer Prepare, and reference proof, then reacquired in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, peer fanout, and reference proof run without either exclusive guard. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation |
| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition |
| v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery |
| v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version |
| v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version |
| Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes |
| Transition commit | Existing object commit locks plus exact source identity and tier-generation checks in the transition path | Publish the exact remote tuple into the matching local version | Publishing a tuple after the source identity or generation changes |
| Transition transaction cleanup | Record validation; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit bucket-lifecycle/physical-set ownership fence or durable takeover CAS | Identity-bound provider probe and deletion of a known canonical candidate | A tier lease alone does not fence the creator. The approved target requires expired ownership, durable takeover, and exact local/source reread before DELETE |
| Transition transaction cleanup | Record validation; expiry check; a next-state ETag CAS for cleanup ownership; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit renewable owner lease or bucket-lifecycle/physical-set ownership fence | Identity-bound provider probe and deletion of a known canonical candidate | The cleanup-state CAS fences a stale predecessor, but the approved target also requires an explicit recovery lease and exact all-pool source reread before DELETE |
| Recovery artifact quota admission (approved target) | Cluster-scoped recovery-admission WRITE lock first; then the one canonical control/source operation lock and any source-protocol metadata/physical locks in that protocol's existing stable order | Bounded internal artifact inventory, exact source/control validation, and create-only installation plus strong readback of one fully encoded export or disposition candidate | Acquiring admission while holding a control, source, disposition, bucket, physical, migration, or decommission guard; remote backend I/O; source-journal deletion; disposition `Applying`; releasing admission before candidate installation/readback converges |
| Manual job | Initial maximum-parity job write; then persisted bucket-level scope create/CAS; later short job/task/result metadata operations | List, checkpoint, append tasks before enqueue, append results after work, renew/take over lease | Holding metadata guards across remote transition PUT; treating the local active-job map as cluster authority. A crash between the job write and scope claim can leave `Running` without a scope record |
| Decommission receipt | Decommission coordinator's source/target record workflow; record-specific conditional writes | Copy/validate durable record, advance receipts, construct and verify expected manifest, conditionally clean exact covered source | Remote tier DELETE; deleting an uncovered or divergent source record |
The tier mutation lock scope is intentionally recorded as **current**, not ideal. Reducing it is allowed only after a durable `Prepared` intent blocks new reference creators across the fleet, existing tier-operation leases drain, and recovery can reconstruct that block without the initiating process. Which network validation can move outside the namespace lock is an **open design**.
Tier mutation backend validation is outside both exclusive guards and is bound to the persisted ETag snapshot. The initiating task is detached from the admin request so cancellation cannot interrupt validation cleanup. Once the coordinator Prepared record and local fence are durable, lease drain, all-node peer Prepare, and reference proof run without the tier-config namespace or `admin_updates` guard. Recovery treats an unexpired Prepared record as active even while the old config ETag remains current. Both guards are reacquired in namespace-then-admin order, and the ETag, candidate digest, exact intent identity, and expiry are revalidated immediately before config CAS. After the coordinator advances to Committed, the guards are released again for peer Commit and local publication.
## Transition transaction
@@ -114,7 +117,7 @@ UploadStarted -> Uploaded -> LocalCommitStarted -> Committed
\-> AbortedNoRemote
```
Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production code emits `CleanupPending` only when recovery probes `UploadOutcomeUnknown` as `UnversionedPresent` or as `VersionedPresent` with a non-nil identifier. The `Uploaded` abort/recovery path deletes its candidate and transaction record directly, and `LocalCommitStarted` mismatch or missing-source recovery retains the record. The `Uploaded` and `LocalCommitStarted` cleanup edges are currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is another current special case: recovery exact-deletes it and removes the record without first persisting `CleanupPending`.
Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production recovery emits `CleanupPending` after an expired `Uploaded` record wins the exact successor CAS, or when an expired `UploadOutcomeUnknown` probe returns `UnversionedPresent` or `VersionedPresent` with a non-nil identifier. `LocalCommitStarted` mismatch or missing-source recovery retains the record; that cleanup edge is currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is retained and never authorizes remote deletion.
The remote candidate itself is named by `canonical_transition_remote_object` under `ilm/transition-transactions/<bucket-hash>/<transaction shards>/<transaction-id>/<write-id>`. That deterministic identity is what a provider probe or exact cleanup must bind; it is distinct from the internal transaction-record key.
@@ -124,13 +127,13 @@ The creator owns the canonical remote candidate until local metadata commits the
| Observed durable state/input | Unique current owner | Current recovery decision | Approved destructive admission |
|---|---|---|---|
| `UploadStarted` | Originating transition attempt; current durable exclusion is incomplete | Retain | No delete. The upload may still publish |
| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery, logically; current record writes do not durably exclude a concurrent worker | Delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
| `UploadStarted` | Originating transition attempt; no current recovery successor is emitted | Retain | No delete. The upload may still publish |
| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery under the exact record generation and recovery lock | Conditionally delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
| `UploadOutcomeUnknown`; probe returns `UnversionedPresent` | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, delete the unversioned candidate, delete the record | Exact transaction/canonical object/backend identity, explicitly unversioned state, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` with a non-nil exact identifier | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, exact-delete that versioned candidate, delete the record | Exact transaction/canonical object/backend identity and remote version, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery | Current code directly exact-deletes that versioned candidate and deletes the record; it does not persist `CleanupPending` | This remains a versioned exact-delete candidate and must not be treated as `UnversionedPresent`. The approved target still requires durable takeover, a current tier lease, and exact reread |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery retains ownership evidence | Retain | A nil identifier is invalid exact-version evidence and never becomes unversioned or remote-delete authority |
| `UploadOutcomeUnknown`; probe ambiguous, unsupported, or errors | Transaction recovery retains ownership evidence | Retain | No destructive action; operator reconcile may inspect after expiry |
| `Uploaded` | Originating transition attempt; current recovery can race it because the persisted fence is not CAS-protected | Current code immediately deletes the candidate and record | **Current safety gap:** approved behavior must first prove the creator cannot still commit by expired ownership plus durable takeover/CAS, then recheck that no matching local commit exists |
| `Uploaded` | Originating transition attempt until expiry; after expiry, the worker that wins `Uploaded -> CleanupPending` by exact ETag CAS | Retain while active; after expiry, persist `CleanupPending`, then recheck and delete the unreferenced candidate or record | Current CAS fences the predecessor, but approved v2 also requires a durable recovery lease, full all-pool source/free-version proof, and before/after fence checks |
| `LocalCommitStarted`; logical source lookup returns `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Current recovery treats this tuple as ownership transfer. The approved target additionally compares recorded source version ID, data directory, modification time, size, and ETag before conditional terminal cleanup |
| `LocalCommitStarted`; logical source is missing, its transition tuple differs, or the read is uncertain | Transaction record/recovery | Retain | No remote delete without a separate durable cleanup proof |
| `CleanupPending`; logical source lookup returns the same current transition predicate | `xl.meta` is remote reachability owner; recovery owns only record cleanup | Delete transaction record | `xl.meta` is owner; do not delete remote. The approved target adds the full recorded source comparison |
@@ -148,12 +151,12 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
### Approved target and open design
- Replace unconditional transaction create/update/delete with create-only, ETag CAS, and conditional terminal delete.
- Preserve the current create-only transaction installation, exact record/ETag successor CAS, and conditional terminal delete, and add mandatory exact-successor strong readback for lost or uncertain responses. No v2 work may regress the existing v1 protections.
- Recovery of `Uploaded` and cleanup-capable states must acquire durable ownership only after the prior owner's expiry. The recovery worker must reread the exact generation after takeover and before remote DELETE.
- Preserve and compare source version ID, data directory, modification time, size, and ETag through local commit and recovery before accepting ownership transfer.
- Recompute and require the exact lowercase sharded path in both transition-transaction and manual-job runtime recovery, and validate a truncated page's continuation token before processing any record from that page.
- **Open:** owner lease duration, clock-skew allowance, takeover revision encoding, and compatibility for existing v1 records that have only `not_after`/`owner_epoch`.
- **Open:** bounded retention and an operator disposition for permanently ambiguous records. Until defined, retained ambiguity is safer than collection.
- The approved v2 owner lease uses the duration, clock-skew allowance, takeover revision, and v1 mapping in [Transition transaction v2 and v1 migration](#transition-transaction-v2-and-v1-migration).
- Active automatic retries are bounded by the recovery-control policy below. Ambiguous evidence becomes explicit `retained_ambiguous` or `operator_required`; source evidence is never collected merely because it is old.
## Tier mutation intent
@@ -163,14 +166,15 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence.
The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. The coordinator then conditionally writes tier config, commits peers, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
### Recovery decisions
| Observed durable state/input | Unique current owner | Current recovery decision | Destructive/config admission |
|---|---|---|---|
| `Prepared`; current tier-config digest equals candidate | Coordinator recovery; each peer recovery owns only its matching peer record/block | CAS to `Committed`, replay peer Commit and publish | Exact mutation identity and candidate digest; never infer from expiry |
| `Prepared`; current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
| `Prepared`; intent is unexpired and current config still proves the old ETag/config | The live coordinator remains owner | Retain `Prepared` and the runtime block | Recovery must not abort work that may still be in lock-free peer Prepare or reference proof |
| `Prepared`; intent is expired and current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
| `Prepared`; config is a third generation, unreadable, or peer outcome is ambiguous | Coordinator record remains owner of the block | Retain `Prepared` and runtime block | No commit, abort, cleanup, or unblock |
| `Committed` | Coordinator recovery, with peers owning convergence of their local records | Replay peer Commit/runtime publication; clean exact converged records | Config ETag/digest and peer identity must match |
| `Aborted` | Coordinator recovery; peer recovery retains the local tombstone | Replay/confirm Abort and clear matching block; retain peer tombstone until expiry plus clock skew and no coordinator | Never roll back config based on timeout alone |
@@ -183,10 +187,39 @@ Intent transitions retry an ETag race at most three times before returning a ret
### Approved target and open design
- Keep create-only, ETag CAS, exact identity comparison, canonical Abort tombstones, and lost-response readback.
- Any shorter configuration-lock window must leave a durable `Prepared` fence installed on every required peer before releasing the broad exclusion scope and must prove recovery restores that fence before admitting reference creators.
- **Open:** the exact split between backend validation, peer fanout, reference scan, config CAS, and publication; expiry must never replace config-generation proof.
- The phase split is fixed: ETag snapshot, lock-free backend validation, ETag revalidation, all-node Prepare, reference proof, ETag/digest/intent revalidation, config CAS, all-node Commit, and local publication. Backend validation uses `(old_config_etag, candidate_digest)` as its stable config generation; the durable mutation identity adds `mutation_id`. Runtime driver revisions are not persistence identities and Add does not require one before publication.
- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof.
- **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records.
## Tier validation probe intent
### Dormant current contract
`TierProbeIntent` defines a strict, checksum-protected `rustfs-tier-probe-intent-v1` envelope and the canonical key `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json`. The same non-nil probe UUID also determines the remote object name `rustfs-tier-probe-<probe-id>`. The path parser rejects uppercase UUID aliases, wrong shards, extra components, and path/payload/object-name disagreement. The durable namespace registry validates this record so pool decommission cannot silently treat it as an ordinary object.
The format binds Add and Edit to the same durable mutation identity tuple `(mutation_id, old_config_etag, candidate_digest)`. The old config ETag is required even for Add because it identifies the complete persisted tier configuration generation, not whether the named destination tier already exists. Verify instead requires the current persisted config ETag and credential-independent backend identity. These are mutually exclusive tagged variants. Every record also requires the tier name, matching destination identity, immutable creator identity/epoch, positive creation time, and an owner fence with nonempty owner, non-nil epoch, and later `not_after` timestamp. In v1 that owner identity must remain exactly equal to the immutable creator identity. It never persists the process-local driver revision or credential-bearing driver fingerprint.
The dormant state graph is:
```text
UploadOutcomeUnknown -> Uploaded -> CleanupPending -> Completed
\-> CleanupPending -> Completed
\-> AbortedNoRemote
```
`UploadOutcomeUnknown` and `AbortedNoRemote` carry no remote version. `Uploaded`, `CleanupPending`, and `Completed` carry either explicit unversioned semantics or one exact nonempty opaque version. Once known, that remote version cannot change. Revision advances by one for each edge: `UploadOutcomeUnknown` is revision 1, `Uploaded` and `AbortedNoRemote` are revision 2, `CleanupPending` is revision 2 or 3, and `Completed` is revision 3 or 4. The strict decoder rejects any other state/revision pairing. The direct `UploadOutcomeUnknown -> CleanupPending` edge is reserved for a future authoritative provider probe that discovers the exact cleanup candidate after the original PUT response was lost.
Create-only, ETag CAS, exact-ETag delete, and read-with-ETag primitives exist for the record, plus a crate-level read-only inspection result that always reports both writer and destructive recovery as disabled. No Add, Edit, Verify, startup loop, periodic loop, admin HTTP route, or remote backend operation currently calls these mutation primitives. Consequently this version creates no records and authorizes no remote PUT or DELETE.
### Activation requirements
- Add/Edit must create the mutation identity before validation and reread the same `(mutation_id, old_config_etag, candidate_digest)` before every probe-intent successor. Verify must reread the same config ETag, tier, and backend identity. A credential rotation may supply usable current credentials only when the persisted destination identity remains exact; it may not weaken operation-generation checks.
- Before enabling any writer, every required node must advertise a probe-intent-specific read/retain/recovery capability. This protocol does not reuse the legacy transition-state reconciliation capability or its token. Unknown or older nodes keep validation in the current process-local mode and no durable v1 record is written.
- The v1 owner fence is immutable and must equal the creator identity. Any future takeover requires a new schema with explicit takeover proof, plus an approved lease duration, clock-skew allowance, durable owner/epoch CAS, and revalidation order. Expiry alone never permits remote DELETE.
- Remote-version discovery and deletion must use the provider-bound, destination-bound implementation and bounded request API approved for that target. Unknown, multiple, changing, unsupported, or unavailable results retain the record.
- A successful or response-lost state write must strongly reread the exact record. Remote DELETE requires a current operation-generation proof, exact destination, current credentials for that same destination, a valid fleet fence, durable takeover, and the same known remote version immediately before and after the call.
- Terminal retention, bounded scanning, metrics, and any HTTP inspect/reconcile route remain unapproved. Raw age is never cleanup evidence, and this dormant core API must not be presented as an operator endpoint.
## Manual transition job, task, result, and checkpoint
### Current contract
@@ -311,7 +344,149 @@ Single and child manifest preparation is bounded by 200,000 journals and a 32 Mi
- New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created.
- Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE.
- Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous.
- **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof.
- Quarantined v1/v2 records, incomplete manifests, and repeatedly failing exact deletes use the bounded retry, explicit operator state, and single-record control surface approved below. Capacity and recovery throughput must not be “fixed” by weakening ownership proof or age-deleting source evidence.
## Bounded recovery control and operator disposition
This section is an **approved target that is not implemented yet**. It closes the ownership, retry, and operator-disposition design required before backlog recovery work can change destructive behavior. It does not authorize an implementation to enable a v2 writer, take over a v1 transaction, or remove a legacy journal until the fleet and storage gates below exist.
### Recovery-control record
Retry scheduling is persisted separately from the source transaction or journal so legacy bytes remain readable and their cleanup ownership does not move. The control record uses schema `rustfs-ilm-recovery-control-v1`, a checksum envelope, a 16 KiB encoded-size limit, strict unknown-field rejection, and this canonical key:
```text
ilm/recovery-controls/<protocol>/<aa>/<bb>/<source-operation-digest>.json
```
Export envelopes use `ilm/recovery-exports/<protocol>/<aa>/<bb>/<export-id>.json`; disposition receipts use `ilm/recovery-dispositions/<protocol>/<aa>/<bb>/<disposition-id>.json`. `export-id` is the SHA-256 of the control ID plus exact observed source content/copy-set digests; `disposition-id` is the SHA-256 of the export ID plus the bounded action. Repeating an identical export or disposition reuses and strongly validates the same canonical object; different bytes at that ID are a conflict. Both use strict checksum envelopes and create-only installation. A control or disposition is limited to 16 KiB. An export is limited to the source protocol's own maximum encoded record size plus 64 KiB for its copy manifest and envelope; it stores the source bytes once rather than duplicating identical replica bytes.
Admission is fail-closed and cluster-wide. One control may have at most one export creation and one disposition application in flight. The immutable candidate is fully encoded before quota admission. The cluster-scoped recovery-admission WRITE lock is always outermost; a caller already holding any control, source, disposition, bucket, physical, migration, or decommission guard must release it and restart in the order above. While admission is held, a complete artifact inventory must prove that the projected totals, including the candidate, are at most 10,000 export envelopes, 10,000 disposition receipts, 1 GiB of encoded export data, and 256 MiB of encoded control/disposition data. The create-only candidate installation and exact strong readback complete before the lock is released, so neither a single candidate nor a concurrent node can oversubscribe the pre-create snapshot. A crash before installation leaves no artifact; a lost create response is resolved by exact canonical readback while admission remains serialized or after reacquiring it in the same order. Replaying an existing canonical ID consumes no new count, byte, or rate token. New creations are limited to ten per authenticated actor per minute and 100 cluster-wide per minute, with at most 32 export creations and eight disposition applications executing cluster-wide. Exceeding a count, byte, rate, or concurrency limit returns a retryable capacity result before source mutation; it never evicts evidence, interrupts an admitted operation, or blocks ordinary object I/O. The collector examines at most 100 terminal artifacts per minute and never acquires the admission lock while holding an artifact/source guard.
`source-operation-digest` is SHA-256 over the length-delimited source protocol, canonical source path, and stable semantic operation identity: transaction UUID, journal identity, or manifest operation ID. For corrupt bytes whose semantic ID cannot be trusted, the canonical path plus a `corrupt` domain separator is the stable identity; a replacement at that path conflicts with the existing control instead of resetting its history. The control's immutable identity repeats the protocol, path, stable identity, and record class. Its mutable `observed_source_generation` contains the source schema, source ETag/content SHA-256, and sorted all-pool copy-set digest. The copy set records each authoritative pool/set identity, canonical path, ETag, byte length, and content SHA-256; an unreachable pool, missing ETag, divergent copy, or incomplete listing cannot produce an actionable generation. The remaining mutable generation contains:
- `revision`, an ETag-CAS successor counter;
- `classification`: `retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`;
- `owner_id`, `owner_epoch`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos` while an attempt is owned;
- monotonic `attempt_count`, `consecutive_failure_count`, `first_failure_at_unix_nanos`, `last_failure_at_unix_nanos`, and `next_attempt_at_unix_nanos`;
- one bounded enum `last_error_code`; no free-form provider error, endpoint, credential material, request payload, or response body is persisted;
- for an operator disposition only, the authenticated actor identifier, reason code, confirmation time, exported payload digest, and exact source/control ETags that were confirmed.
The control record is a scheduler and audit fence, not a remote-object owner. It never substitutes for the source record's version, backend, manifest, source-identity, or absence proof. Before every source CAS, local cleanup, or remote request, the worker strongly rereads every authoritative source copy and the control, requires the current observed generation to match, and then applies the source protocol's own locks and proof. A missing, stale, corrupt, divergent, or unavailable control read cannot authorize work.
Control creation is `If-None-Match: *`; every update is exact ETag `If-Match`; response loss requires exact strong readback. A legal source-state CAS does not create a new control. The stable control CAS advances `observed_source_generation` only from the exact predecessor to one source-protocol successor while preserving `first_failure_at_unix_nanos`, lifetime `attempt_count`, and first-seen lineage. If the source CAS succeeded before a crash, recovery accepts the new bytes only after validating that exact legal successor and then converges the old control generation by CAS. A multi-edge jump, semantic-identity change, replacement ETag/content, or unavailable predecessor proof is a conflict and cannot reset counters. Recovery conditionally removes a terminal control only after strongly proving the stable source operation resolved or absent, recording any active decommission terminal receipt, and confirming that no in-flight operator request still names its ETag. `durable_namespace.rs` must register every new namespace, path parser, decoder, size bound, successor relation, and terminal checkpoint before rollout.
### Transition transaction v2 and v1 migration
The successor envelope is `rustfs-transition-transaction-v2`. It preserves the v1 transaction ID, deployment ID, write ID, complete source identity, tier/backend identity, canonical remote object, remote-version state, operation state, and revision. It also records an immutable `origin_format` (`native_v2` or `migrated_v1`), the state-entry revision and predecessor state/revision, and, for migration, the exact source v1 state/revision. These fields make the state history needed for destructive authorization independently checkable instead of inferring it from the current state. The envelope and body reject unknown fields, checksum every field, use no default for a required v2 value, require the existing exact lowercase canonical path, and reject nil UUIDs, nonpositive timestamps, revision zero/overflow, lease inversion, impossible state/revision history, and illegal state/version combinations. It replaces the fixed ownership pair with:
- immutable `creator_epoch` and `creator_not_after_unix_nanos`, copied exactly from v1 `owner_epoch` and `not_after_unix_nanos` during migration;
- optional `created_at_unix_nanos`; migrated v1 records use `None` rather than inventing an age;
- mutable `owner_role` (`creator` or `recovery`), `owner_id`, `owner_epoch`, `owner_generation`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos`. `owner_id` is the authenticated stable fleet-node identity; `owner_epoch` is a fresh UUID for one process claim. A node without both identities cannot create, renew, take over, or act on v2.
A native v2 create uses revision and owner generation 1, a fresh non-nil creator owner/epoch, `UploadStarted`, and an unknown remote version. The first rollout keeps the current seven-day creator ownership window. Creator leases are not renewable; a creator that cannot finish inside the safety window stops publishing and leaves the record for recovery. Recovery-owner leases are 15 minutes. The persisted clock-skew allowance is five minutes: an action may start only when its bounded deadline fits before `lease_expires_at - 5 minutes`, and another owner cannot take over until its local time is at least `lease_expires_at + 5 minutes`. A recovery attempt remains capped at five minutes and every remote call remains subject to its narrower client deadline. A recovery renewal is a same-owner, next-revision CAS that strictly extends expiry; a takeover changes `owner_role`, `owner_id`, and `owner_epoch`, increments `owner_generation` and `revision`, and uses the exact observed ETag. Takeover and state advancement are separate CAS operations; one revision cannot both acquire ownership and claim a recovery outcome.
Before migration can be enabled, the fleet must first deploy a v1 creator fence that strongly rereads the exact transaction path, ETag, state, owner epoch, and source generation immediately before local metadata publication and refuses to publish after any migration/takeover change. The fleet then durably disables new v1 admission and proves that every captured creator/recovery process epoch has either acknowledged quiescence or terminated. A paused or unreachable epoch prevents migration. This drain barrier is distinct from format capability advertisement and remains in force until v2 writer admission is enabled.
Only these checksum-valid v1 state/revision pairs are migration inputs: `UploadStarted@1`; `UploadOutcomeUnknown@2`; `AbortedNoRemote@2`; `Uploaded@2` or `Uploaded@3`; `LocalCommitStarted@3` or `LocalCommitStarted@4`; `Committed@4` or `Committed@5`; and `CleanupPending@3`, `CleanupPending@4`, or `CleanupPending@5`. Any other pair is `corrupt`, inspect-only, and cannot be migrated or authorize a probe, local cleanup, or remote DELETE. A native v2 record must prove a legal predecessor edge at its recorded state-entry revision; ownership-only revisions may increase the outer revision but cannot change the recorded state-entry history. Missing, contradictory, or skipped history is corrupt.
An identity-preserving `v1 -> v2` conversion is one legal successor with `revision + 1` and an unchanged remote tuple. The state is unchanged except that every v1 `UploadStarted` maps conservatively to v2 `UploadOutcomeUnknown`. Historical v1 writers could issue PUT while still in `UploadStarted`; no age, fleet version, or current process observation proves that a retained record came from the later pre-PUT-fence writer. It is permitted only when:
1. every node that can create, commit, recover, heal, or decommission the record advertises both `transition_transaction_v2` and `ilm_recovery_control_v1` for the captured fleet/topology generation, the durable v1-admission stop is active, and the process-epoch drain barrier above is complete;
2. current time is at least the v1 `not_after_unix_nanos` plus five minutes of skew;
3. the canonical path, checksum, full immutable identity, state, remote-version invariant, source record, and ETag all match the observed v1 generation;
4. the migration CAS and strong readback install one fresh recovery lease before any state transition or side effect.
The original creator must successfully CAS `UploadStarted -> UploadOutcomeUnknown` before issuing remote PUT, and must CAS the exact current owner generation to `LocalCommitStarted` before publishing local metadata. A v2 takeover therefore fences a delayed creator. An implementation that can issue PUT or publish after losing this CAS is not compatible with this protocol.
After takeover, recovery applies this matrix:
| State | Approved recovery after exact takeover | Required proof before side effect |
|---|---|---|
| native-v2 `UploadStarted` | CAS `AbortedNoRemote`, then conditionally remove the terminal transaction | Valid native-v2 history proves the creator was fenced before the mandatory pre-PUT `UploadOutcomeUnknown` CAS; migrated v1 never enters this row and no remote request is made |
| `UploadOutcomeUnknown` | Probe under the exact backend lease. Missing becomes terminal cleanup; proven unversioned presence or one nonempty, non-nil exact version becomes `CleanupPending`; nil, ambiguous, or unsupported results become `retained_ambiguous` | Exact transaction/control generations, bounded live probe, current tier destination and lease |
| `Uploaded` | If the exact transitioned tuple or its free-version owns the candidate, remove only the transaction. If the complete original source is still unchanged and no local commit/free-version exists, CAS `CleanupPending`. Otherwise retain | All-pool source/free-version read, full source tuple, bucket incarnation, tier generation, object locks, and post-probe revalidation |
| `LocalCommitStarted` | Exact committed tuple/free-version means record-only cleanup. A fully unchanged original source with no partial committed tuple may move to `CleanupPending`. Missing, divergent, partial, or unavailable metadata is `retained_ambiguous` | Same all-pool proof, including data directory, modification time, size, ETag, transition transaction ID, remote tuple, and destination identity |
| `CleanupPending` | Resume the same exact idempotent candidate delete, or remove only the transaction when local ownership transfer is proven | Current owner lease, source/free-version proof, exact tier lease, physical locks, and before/after fence checks |
| `Committed` | Conditionally remove only the exact terminal transaction when complete local ownership transfer is proven; otherwise classify the record as corrupt or retain it as ambiguous | All-pool strong read matches the complete logical `xl.meta` source identity, transaction ID, remote tuple, destination identity, and any required decommission terminal receipt; no remote DELETE |
| `AbortedNoRemote` | Conditionally remove the exact terminal transaction | Valid native-v2 pre-PUT history or exact migrated v1 `AbortedNoRemote@2`, exact terminal generation, and any required decommission terminal receipt; no remote DELETE |
Remote DELETE is never admitted directly from `UploadStarted`, `UploadOutcomeUnknown`, `Uploaded`, or `LocalCommitStarted`; it first requires a CAS-protected `CleanupPending` generation with known remote-version semantics. A lost source read, mixed all-pool result, expired lease, failed renewal, or source/control CAS conflict retains the evidence and performs no destructive action.
New writers emit v2 only after the homogeneous fleet gate, durable v1-admission stop, and process-epoch drain barrier are complete. During a rolling upgrade, new readers accept v1 but all writers continue v1 and no v1 takeover/migration occurs. A v1 reader rejects and retains v2. Downgrade is blocked until v2 creation is disabled and all v2 transactions and recovery-control records are drained or exported; live v2 bytes are never rewritten to v1.
### Retry and retention policy
An attempt that loses a CAS or discovers a newer source generation reloads instead of recording a remote failure. A retryable transport timeout, backend 5xx/throttle, metadata quorum outage, or bounded remote-delete failure increments the persisted counters and schedules:
```text
min(60 seconds * 2^min(consecutive_failure_count - 1, 6), 1 hour)
```
A deterministic multiplier from 80 to 100 percent, derived from the source-generation digest and attempt count, is applied to that capped base. The jitter can only shorten the delay and therefore never exceeds the one-hour cap; restarts reproduce the same deadline without synchronizing a fleet. Success or a proven source-state advance resets `consecutive_failure_count` but never decreases `attempt_count`. `next_attempt_at_unix_nanos` is only a not-before scheduler hint; ownership and destructive authority still require the lease and source proofs.
After 32 consecutive retryable failures or seven days since `first_failure_at_unix_nanos`, whichever occurs first, the control CAS moves to `operator_required` and automatic attempts stop. Unsupported probes and unknown remote-version semantics move directly to `retained_ambiguous`; corrupt source bytes use `corrupt`; incomplete destructive evidence uses `operator_required`. None is periodically hot-looped. An operator may explicitly request another bounded attempt after the underlying capability or configuration changes, but the request creates a new owner lease and preserves the lifetime attempt count.
This policy bounds automatic work, not evidence lifetime. A source transaction, journal, or manifest is never deleted solely because it is old, numerous, or over a byte threshold. `operator_required` source evidence remains until its protocol reaches a proven terminal state or the legacy-journal disposition below is completed.
Resolved control tombstones are retained for at least 30 days, immutable export envelopes for at least 90 days, and compact completed disposition receipts for at least 365 days. A collector may conditionally remove only a terminal artifact past its floor after proving that the bound source generation is absent where required, no nonterminal successor or active decommission references it, and the terminal audit checkpoint is durable. Capacity pressure blocks new export/disposition work rather than evicting unexpired or nonterminal evidence. Uncertainty retains the artifact; collection never authorizes source or remote deletion.
### Legacy journal and manifest disposition
Journal v1 has neither backend identity nor remote-version authority; v2 has backend identity but still lacks remote-version semantics. Their automatic classification is `retained_ambiguous`, and neither recovery nor an operator action may instantiate a backend or issue remote PUT, GET, probe, or DELETE from those bytes. The approved single-record actions are:
- **inspect**: strictly decode a server-reconstructed canonical journal identity, perform an all-pool strong read, and return a redacted copy-set/content digest, version, quarantine reason, control classification, age information when known, topology readiness, and decommission coverage; it changes nothing and does not return raw object/version fields by default;
- **export**: after a fresh exact inspect, create-only persist an immutable `rustfs-ilm-recovery-export-v1` envelope containing the raw source bytes and sorted copy manifest, then strongly read it back. The response downloads that envelope rather than rereading the live journal, uses no-store/attachment semantics, and never adds credentials or backend configuration;
- **abandon after export**: v1 or v2 only; create a `Prepared` `rustfs-ilm-recovery-disposition-v1` receipt bound to the immutable export and every source copy, conditionally remove only those exact local journal generations, prove every bound copy absent with no replacement, and advance the receipt through `Applying` to `Completed`. This accepts a possible remote storage leak and never asserts that cleanup occurred.
Export and abandon require a fresh all-member capability/topology proof including each member's current process epoch; inspect may remain available in a mixed fleet but returns not-ready for mutation. `abandon after export` uses POST and requires `confirm: true`, `action: abandon_remote_cleanup`, `acknowledge_remote_cleanup_abandoned: true`, the export operation ID/digest, source content and copy-set digests, every source ETag, control ETag, and a bounded operator reason code. It is refused while an active decommission or migration receipt covers either record, while physical copy discovery is incomplete, or when the implementation cannot target every discovered copy with its own `If-Match` condition.
The disposition receipt has immutable action/export/control identities and an immutable sorted copy manifest. Its ETag-CAS generation contains state `Prepared`, `Applying`, or `Completed` and a monotonic sorted `confirmed_absent` set naming only entries from that manifest. Before `Prepared -> Applying`, a fresh all-pool read must find every bound copy at its exact ETag/content digest and repeat the fleet, lock, migration, and decommission checks. The manifest can never be widened, reordered, or replaced.
During `Applying`, recovery treats each manifest entry independently while retaining the original all-pool boundary. An entry already in `confirmed_absent` must still be strongly absent with no successor or replacement generation. For an unconfirmed entry, an exact ETag/content match may be conditionally deleted and then added to `confirmed_absent` only after strong absence readback. If a crash or lost response left that exact path absent before the progress CAS, recovery may add it only after the same strong absence, stable source/control generation, topology, process-epoch, migration, and decommission proofs establish that no replacement exists. A different ETag/content, an unbound copy, an unreadable member, or loss of any proof is a conflict and preserves the current progress. Thus a crash after deleting copy A but before recording its progress can converge and continue with copy B without requiring deleted copy A to reappear.
Immediately before each local metadata deletion, the server repeats the applicable all-pool/fleet/lock checks and conditionally targets only the still-unconfirmed exact ETag. Completion requires every immutable manifest entry in `confirmed_absent`, a fresh all-pool proof that all remain absent without replacement, unchanged fleet/process epochs, and no active decommission or migration coverage. A lost final response is success only when strong readback proves the canonical receipt `Completed`. Recovery may repeat only this canonical operation and never creates a tier client or issues a backend request.
Malformed or unsupported bytes whose outer v1/v2 identity cannot be proven are inspect-only and cannot use abandon. Versions v3-v6 never use `abandon after export`. Their known candidate or manifest ownership must converge through the normal exact protocol. An operator may inspect/export and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or remote-version validation. `Preparing`/`Aborting` manifests may use their existing whole-set rollback; `DispatchAuthorized`/`Completed`, a missing member, a nonempty operation namespace, or any uncertain binding cannot be manually removed.
### Admin and metrics contract
The approved surface is single-record and uses a protocol-specific expected tuple; it does not reuse the legacy metadata-reconcile digest or create a bucket/prefix job:
```text
GET /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
GET /rustfs/admin/v3/ilm/recovery/records/<control-id>
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
GET /rustfs/admin/v3/ilm/recovery/exports/<export-id>
```
List and redacted inspect require `admin:ListTier`. Raw export creation/download, retry, or abandon require `admin:SetTier` because legacy bytes can reveal bucket, object, tier, and remote-version information. The default page is 100 records and the hard maximum is 1,000. A truncated page without a continuation marker is an error, and counts from an incomplete scan are labeled incomplete rather than reported as zero or complete.
Inspect returns a 15-minute opaque observation receipt bound to the authenticated actor, canonical record identity, source/copy ETags and digests, topology/fleet generation, every member process epoch, action class, issue/expiry time, and nonce. It is observation evidence, not mutation authority. POST requires that receipt and `confirm: true` for terminal actions, and binds the control ETag/revision/classification, requested action, and export digest when applicable. The server repeats all live proofs; a restarted member, membership change, or process-epoch mismatch invalidates the receipt. Client fields are concurrency guards, not authority. Export download reads the immutable envelope, uses TLS plus `Cache-Control: no-store` and attachment disposition, and never logs the raw payload.
Metrics use bounded labels only:
- `rustfs_ilm_recovery_records{protocol,classification,schema}` and `rustfs_ilm_recovery_oldest_age_seconds{protocol,classification,schema}`;
- `rustfs_ilm_recovery_attempts_total{protocol,outcome,error_code}`;
- `rustfs_ilm_recovery_operator_actions_total{protocol,action,outcome}`;
- scan completeness, corrupt-record, and orphan-control counters.
Every label value is a closed enum. `schema` exposes recognized schema identifiers only; an unrecognized raw value maps to `unknown`, while a recognized future schema disabled by the current fleet maps to `unsupported`. `protocol`, `classification`, `outcome`, `error_code`, and `action` likewise map unknown input to one bounded fallback and never expose decoded or operator-provided text.
Object names, bucket names, tier names, transaction IDs, control IDs, endpoints, ETags, error text, and credentials are never metric labels. Admin output may identify the selected record but redacts credentials and raw backend configuration. Audit/log events use the repository ILM event fields, stable reason codes, authenticated actor, source/control generations, action, and outcome; they do not persist or log provider response bodies.
One canonical source generation counts once regardless of its physical copy count or how many recovery passes observed it. Attempt counters increment once per coordinator attempt, not per replica, CAS retry, or page revisit. Aggregate totals and oldest age are authoritative only after a complete all-pool scan; partial coverage reports `incomplete` and never publishes a false zero. A legacy record's first-seen time comes from its durable control record rather than an inferred object modification time.
### Required protocol fixtures
Implementation acceptance requires deterministic crash/restart and mixed-version fixtures, not timing-only tests. At minimum they cover:
- a historical v1 `UploadStarted@1` whose PUT may have reached the provider, proving migration yields `UploadOutcomeUnknown` and never `AbortedNoRemote` or a direct delete;
- every accepted v1 state/revision pair above plus checksum-valid impossible pairs such as `CleanupPending@1`, proving impossible history is inspect-only and makes zero backend calls;
- an in-flight v1 creator interleaved with admission stop, process-epoch drain, migration, takeover, and local publication, proving no stale creator can publish after takeover;
- `Committed` with complete, missing, partial, divergent, and unavailable all-pool `xl.meta` ownership proof, proving only the complete exact tuple permits record cleanup;
- an old reader retaining v2, a mixed fleet blocking v2 writer and migration, and downgrade refusing until v2/control records are drained or exported;
- operator abandon across a crash after one per-copy delete, a lost delete response, a lost progress CAS, a replacement ETag, incomplete topology, and active decommission, proving progress is monotonic, replacements survive, unsafe cases retain evidence, and every case issues zero backend PUT, GET, probe, or DELETE calls;
- canonical export replay, a crash before candidate installation, a lost create response, concurrent admission at the remaining-byte boundary, and actor/cluster count, byte, rate, and concurrency exhaustion, proving duplicate IDs consume no new quota, projected totals never oversubscribe, and admission failure mutates no source evidence.
## `xl.meta` free-version boundary
@@ -417,7 +592,7 @@ A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a
Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current exact ETag conditional-delete primitive. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target.
@@ -427,7 +602,7 @@ Receipts have no enum state. Their legal evolution is `absent -> checkpoint -> m
| Observed state | Unique current owner | Current recovery decision | Destructive admission |
|---|---|---|---|
| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. An eligible v6 journal/manifest record is directly removed by exact ETag; transition-transaction cleanup follows its documented current unconditional path | Receipt state grants no remote-delete authority |
| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. Eligible v6 journal/manifest and transition-transaction records are removed only by their exact observed ETag | Receipt state grants no remote-delete authority |
| Source and target exact identity/checkpoint agree | Decommission coordinator for the run token | Create or CAS-advance the run-scoped receipt | Successor must be monotonic and topology-bound where required |
| Receipt already covers the same successor | Decommission coordinator for the run token | Treat as idempotent | Exact identity/checkpoint only |
| Conflicting receipt, checksum/schema/path error, divergent target record, missing ETag, or non-successor checkpoint | No decommission actor acquires cleanup authority | Fail decommission and retain source | Never overwrite or guess |
@@ -451,7 +626,7 @@ Receipt and expected-manifest create/CAS conflicts retry at most three times. Ex
### Approved target failure matrix
The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. In particular, transition-transaction initial and successor writes and deletes are currently unconditional, while the transition transaction's initial write and the manual job's initial write have neither create-only installation nor mandatory lost-response strong-readback convergence.
The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. Transition transaction v1 now has create-only installation, exact ETag successor CAS, and conditional terminal deletion, but it still lacks mandatory lost-response exact-successor readback and a renewable durable recovery lease. The manual job's initial write remains non-create-only.
| Event | Approved result |
|---|---|
@@ -462,19 +637,20 @@ The matrix below is the normative approved target, not a blanket description of
| Crash after remote DELETE but before journal/free-version cleanup | Retry the same exact idempotent DELETE under the same fences, then conditionally clean local evidence |
| Cancellation | Stop issuing new work, persist monotonic cancellation where the protocol has it, and leave ambiguous durable records for recovery. Cancellation is never rollback proof after authorization |
| Rolling upgrade | Gate writers on the minimum capability required by the format. Known older journal/RPC versions follow their explicit compatibility rule; unknown formats are retained |
| Downgrade | Drain v6 journals before removing all v6-aware workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
| Downgrade | Drain v6 journals and any enabled transition-v2/control protocol before removing their capable workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
| Corrupt or unknown input | Record a diagnosable failure, retain bytes, and block destructive action/completion |
Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have a complete persisted-format negotiation for rolling downgrade. Until one is designed, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have an implemented persisted-format negotiation for rolling downgrade. The approved transition-v2/control gate above is not current behavior. Until the applicable gate is implemented, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
### Current format compatibility decisions
| Family/version | Current reader and writer behavior | Upgrade, downgrade, and ignore rule |
|---|---|---|
| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. A future schema needs a fleet writer gate and an old-reader retention test before rollout; downgrade behavior is open |
| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. V1 remains the only writer format until the approved v2 fleet gate is implemented; a v2 reader never rewrites an active v1 record |
| Transition transaction v2 and recovery-control/export/disposition v1 | Approved target only; no current reader or writer emits these formats | Roll out read support before the homogeneous writer gate; old readers reject and retain. Disable creation and prove all active records drained before downgrade; never rewrite v2 to v1 |
| Tier mutation intent v1; peer RPC v3/v4 | Durable readers/writers require intent v1. New peers accept signed/canonical v3 and v4 RPC; old v3 peers return an exact authenticated unsupported response to v4 | Pause and drain edit/remove/clear across the mixed interval; do not automatically retry v4 as v3. Unknown durable intent is retained and blocks recovery |
| Manual job/scope/task/result v1 | Writers emit the v1 family. Manual-job runtime recovery accepts an uppercase UUID path when both shard strings match its uppercase prefix, then loads the lowercase canonical job by UUID; the decommission validator recomputes the canonical path and rejects that alias. Other decoder/path/checksum failures stop reconciliation. Runtime capabilities advertise `enqueue_only` and `async`, but the async run handler does not consult a fleet capability gate and a direct request creates a job | Runtime recovery still needs exact lowercase canonical-path validation to prevent alias-driven duplicate work. Caller/operator orchestration must verify every required node and fail closed when capability is unknown or unsupported. An automatic server-side fleet gate and persisted downgrade negotiation remain open; unknown records are never ignored as completed work |
| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled |
| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Never translate empty version ID to known-disabled or authorize remote DELETE. Retain unless the approved exact inspect/export/abandon protocol conditionally removes only the local journal generation |
| Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof |
| Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization |
| Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation |
@@ -484,10 +660,10 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr
| Protocol | Current operator/telemetry surface | Current retention | Required follow-up |
|---|---|---|---|
| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Backlog age/count/state metrics, bounded policy, and durable takeover status |
| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Implement the approved v2 lease/takeover, recovery-control status, bounded retry, and backlog metrics |
| Tier mutation intent | Admin mutation response plus recovery diagnostics; no dedicated reconcile API | Peer aborted tombstone through expiry plus skew; ambiguous coordinator/peer records retained | Status/reconcile view for mutation, peer convergence, config generation, and blocked tiers |
| Manual job | POST run response, GET status, DELETE cancel; runtime capabilities advertise both modes | Job/task/result history is indefinite. Terminalizers only best-effort delete the exact scope; startup skips a terminal job with a leftover scope, which remains until a later admission claimant lazily replaces it | Age/count/bytes limit and a terminal-history/scope GC protocol that preserves recovery evidence |
| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Safe operator inspection/disposition, backlog age/count by version/state, bounded recovery without evidence loss |
| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Implement the approved single-record inspect/export/disposition, bounded retry controls, and logical backlog metrics |
| Decommission receipt | Decommission state/events including `receipt_cleanup_failed` | Completion triggers only best-effort receipt/manifest cleanup. Delete failures reported as `receipt_cleanup_failed`, as well as abandoned runs, can leave run-scoped records behind | Run-scoped retention and resume-safe cleanup policy |
Retention is a protocol transition, not raw deletion. Any collector must name its unique owner, minimum age/count/bytes bound, exact terminal or quarantine predicate, readback behavior, decommission interaction, and audit/metric output. It may not collect a record solely because it is old.
docs/architecture/remote-credential-sealing-adr.md
+124 -25
View File
@@ -1,52 +1,151 @@
# Remote Credential Sealing ADR
**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, its fail-closed rules, and the mixed-version compatibility matrix.
**Source of truth:** the three stores that hold remote credentials today — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, and `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — plus the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier.rs`, and `crates/ecstore/src/bucket/on_demand_migration/config.rs`.
**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another.
**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `crates/ecstore/src/bucket/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
## Recommendation
Seal the credentials, ship the write side off by default, and claim a security benefit only for deployments running the Vault Transit or AWS KMS backend — everywhere else recommend full-disk encryption and the fail-closed parse fix below, which cost no code and cover strictly more.
## Whether encryption buys anything here
This decides the whole question, so it comes before the design. Sealing converts "read the drives" into "read the drives **and** hold an authenticated path to the key". How much that is worth depends entirely on the KMS backend, and [../operations/kms-backend-security.md](../operations/kms-backend-security.md) is explicit about the difference.
| Backend | Where the key that unwraps these credentials lives | What sealing is worth |
|---|---|---|
| Vault Transit, AWS KMS | Inside Vault or AWS; only ciphertext ever leaves | Real. An offline copy of `.rustfs.sys` is inert. Each unwrap is a live authenticated call that is logged, rate-limitable and revocable, and revoking the node's identity retroactively protects every copy already taken |
| Vault KV2 | In Vault KV v2, Base64-encoded, not wrapped | Thin. The referenced document states that KV read access is equivalent to holding the master keys, so the boundary is the Vault ACL on the key prefix — worth something only when that ACL is genuinely narrower than access to the drives, and worth nothing against anyone holding both |
| Local, Static | In `key_dir` on the node's own filesystem, or in the process environment | Close to nothing. Whoever reads the drives on a node usually reads the host too. The only gap it covers is media taken away from the host — the same gap full-disk encryption covers better |
Two honest limits hold on every backend. Sealing is **not** a defense against code execution on a node: the sealer runs in-process on every node that has to build a remote client, so an attacker at that level asks it to unseal and gets the plaintext. And it is not a defense against an authorized admin, because an admin who can rewrite a target can point it at a remote they control instead of reading the old secret.
What it does remove is the media-level read: a decommissioned or RMA'd drive, a drive-level backup or volume snapshot, a host path exposed by a bad mount, a copy of a drive taken for support. That threat is real, and it is the only one this design addresses.
## Decision
Remote credentials are sealed **per field, into an added field, behind one shared seam**, and ECStore reaches KMS through an installed hook rather than a crate dependency.
1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr`, and `OnDemandMigrationSys` seal and unseal through a single ECStore-owned envelope type. No consumer talks to KMS, and no consumer defines its own ciphertext layout.
2. **Only secret material is sealed.** `secret_key` and `session_token` are sealed. Endpoint, region, ARN, bucket, prefixes, path style, TLS flags, and the custom CA bundle stay in clear text: they are needed for validation, listing, and support diagnosis, and none of them is a secret.
3. **Sealed material lives in an added field, never in place of the plaintext field.** A record carries either the plaintext field or the sealed field. A reader that does not understand the sealed field therefore finds the credential *absent* rather than finding a ciphertext string it would sign requests with.
4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the single point that needs plaintext, so admin reads, listings, validation, and status paths never call KMS.
1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr` and `OnDemandMigrationSys` seal and unseal through `crates/ecstore/src/bucket/sealed_credentials.rs`. No consumer talks to KMS, and no consumer defines its own ciphertext layout.
2. **Only secret material is sealed,** and the enumeration comes from the redaction code, not from a pair of field names — see [Which fields are sealed](#which-fields-are-sealed). Endpoint, region, ARN, bucket, prefixes, path style, TLS flags and the custom CA bundle stay in clear text: they are needed for validation, listing and support diagnosis, and none of them is a secret.
3. **Sealed material lives in an added field, and the plaintext field is emptied rather than removed.** A reader that does not understand the sealed field must see a credential that is *present and empty*, so it takes a missing-credential path rather than a parse failure. Removing the field instead is what turns this design into an outage; [Compatibility](#compatibility-per-store-because-the-three-differ) explains why.
4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the only place that needs plaintext, so admin reads, listings, validation and status paths never call KMS — and a KMS outage never changes which targets or tiers *exist*.
## What is stored today, and where
Two of the three are not files at all. `bucket-targets.json` and `on-demand-migration.json` are named sub-configurations inside one msgpack blob per bucket, and only the tier configuration is its own object.
| Store | Reached as | Actually persisted at | Written by | Container |
|---|---|---|---|---|
| Replication and ILM targets | `BUCKET_TARGETS_FILE` | `BucketMetadata::bucket_targets_config_json`, msgpack field `BucketTargetsConfigJSON` | `BucketMetadata::update_config`, then `BucketMetadata::save_with_store`; `crates/ecstore/src/bucket/metadata_sys.rs` serializes the update under a transaction lock | `{BUCKET_META_PREFIX}/{bucket}/{BUCKET_METADATA_FILE}` in `RUSTFS_META_BUCKET` (`crates/ecstore/src/disk/mod.rs`) |
| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; `update_config` additionally refuses a blob this build cannot parse | same blob as above |
| Remote tiers | `TIER_CONFIG_FILE` | its own object, a four-byte `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header followed by an `rmp_serde` payload of `ExternalTierConfigMgr` | `TierConfigMgr` through `encode_external_tiering_config_blob`, under `tier_config_lock_path` | `tier_config_path` under `CONFIG_PREFIX` in `RUSTFS_META_BUCKET` |
The consequence of the first two sharing a blob is that any change to how that blob parses has a blast radius covering policy, lifecycle, versioning, object lock and everything else in `BucketMetadata` — not just credentials.
## The at-rest boundary as it stands
Three things hold the line today, and all three keep working whether or not sealing ships.
- **The reserved bucket.** `RUSTFS_META_BUCKET` is `.rustfs.sys`; `is_reserved_or_invalid_bucket` keeps it off the S3 surface, and the admin inspect archive in `rustfs/src/admin/handlers/inspect_archive.rs` runs its request through a strict bucket-name check that a dot-prefixed reserved name does not pass.
- **Admin authorization** on every route that can read or write one of the three configurations.
- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
So no API returns a stored secret. The bytes are reachable by reading the drives, and that is the boundary sealing is proposed to move.
## Which fields are sealed
The authoritative list of what this codebase treats as secret is the redaction functions above, and it is wider than `secret_key` plus `session_token`.
| Store | Sealed | Left in clear text although redacted |
|---|---|---|
| Targets | `Credentials::secret_key`, `Credentials::session_token` | — |
| On-demand migration | `SourceCredentials::secret_key`, `SourceCredentials::session_token` | — |
| Tiers | `secret_key` on each of the nine S3-family backends in `crates/ecstore/src/services/tier/tier_config.rs`, `TierAzure::sp_auth.client_secret`, and `TierGCS::creds` | `TierS3::aws_role_web_identity_token_file`, which is a path rather than a secret |
`TierGCS::creds` carries a whole service-account key and is the largest single secret of the three stores; a design that sealed only fields literally named `secret_key` would leave it in clear text. `aws_role_web_identity_token_file` points at a file outside `.rustfs.sys`, so sealing it would protect nothing — and a tier configured that way stores no long-lived secret at all, which is the cheapest mitigation available and should be preferred where the remote supports it.
## Envelope format
A versioned, self-describing record: envelope version, KMS key id, KMS key version, algorithm, nonce, and ciphertext. It is stored base64 in the two JSON stores and as raw bytes inside the msgpack payload of the tier blob; the tier blob's own `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header constants are unchanged, because the envelope carries its own version.
`SealedCredential` in `crates/ecstore/src/bucket/sealed_credentials.rs`: envelope version, KMS key id, optional KMS key version, algorithm label, and the ciphertext produced by the sealer. It is stored base64 in the two JSON stores and as bytes alongside the tier payload. `SEALED_CREDENTIAL_VERSION` is checked by `SealedCredential::check_version` *before* the sealer is consulted, so an envelope from a newer build is refused here rather than inside a backend.
The KMS encryption context binds each ciphertext to the record that owns it — store kind, owning bucket or tier name, and field name — so a ciphertext copied into another bucket, another tier, or another field fails to decrypt instead of silently authorizing a different remote.
The encryption context binds each ciphertext to the record that owns it. `SealScope` renders store kind, owner (bucket name, tier name or target ARN) and field name into the context, so a ciphertext copied into another bucket, another tier or another field fails to decrypt instead of silently authorizing a different remote. Those context keys are part of the on-disk contract: changing one makes every existing ciphertext undecryptable.
The envelope deliberately does **not** carry its own scope. A scope read out of the stored bytes would be attacker-controlled, and checking a ciphertext against a context it supplied itself proves nothing. The scope is always re-derived from where the ciphertext was found, which is also a constraint on any rewrap job — see [Rotation](#rotation).
## Why a hook instead of a dependency
`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering. The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup: `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs`, installed by `install_ecstore_event_dispatch_hook` in `rustfs/src/server/event.rs`, and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`. Sealing uses the same shape, with the binary supplying an implementation backed by `get_global_kms_service_manager` in `crates/kms/src/service_manager.rs`.
`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
## Compatibility matrix
## Compatibility, per store, because the three differ
| Stored form | Reader | Behavior |
|---|---|---|
| Plaintext (today's format) | Old node | Unchanged. |
| Plaintext | New node | Read as plaintext, no KMS call. Carries a `RUSTFS_COMPAT_TODO` marker per [compat-cleanup-register.md](compat-cleanup-register.md). |
| Sealed | New node, hook installed | Unsealed at client construction. |
| Sealed | New node, no hook or decrypt failure | Typed error; the target, tier, or source is unusable and reports why. Never a default, an empty credential, or the ciphertext bytes. |
| Sealed | Old node | The credential field is absent, so the old node fails closed on its existing "missing credentials" path. This is the migration hazard the rollout gate exists for. |
The generic matrix is short: a plaintext record reads unchanged on any node; a sealed record reads on a new node with a sealer installed; a sealed record on a new node without one is a typed error and never a default. Everything difficult is in what an **old** node does, and the three stores behave differently enough that a single answer would be wrong.
## Rollout gate
| Store | Old node meets an added sealed field | Old node meets an emptied plaintext field | Verdict |
|---|---|---|---|
| Targets | Ignored. `BucketTarget` and `Credentials` do not use `deny_unknown_fields` | `Credentials` has no struct-level `serde(default)`, so a **missing** `secretKey` is a hard parse error for the whole document — but an **empty** one parses | Safe only if the plaintext field is emptied rather than removed |
| On-demand migration | **Rejected.** `OnDemandMigrationConfig`, `SourceConfig` and `SourceCredentials` all carry `deny_unknown_fields`, so the whole configuration becomes unreadable, and `BucketMetadata::update_config` also refuses to persist it | Parses | Needs a reader-first release before any node writes the field |
| Tiers | The payload is compact `rmp_serde`, which encodes structs positionally; an added field is an arity change a reader built for the previous struct cannot skip. `decode_external_tiering_config_blob` also rejects any `TIER_CONFIG_VERSION` it does not know | Parses | The sealed value must not be added to any struct inside the existing payload |
Sealing is written only when KMS is configured **and** a module switch in `rustfs/src/module_switches.rs` is on, defaulting off in the release that introduces it. Reading sealed records is always supported; writing them is what waits. Operators enable the switch after every node in the cluster can read the format, and existing plaintext records are sealed by re-submitting the configuration through its admin API — this task ships no in-place migration sweep.
Two of those rows are load-bearing enough to spell out.
**Targets.** `BucketMetadata::parse_all_configs` responds to an unparseable `bucket-targets.json` by logging `bucket_metadata_parse_failed` and setting `bucket_target_config` to `BucketTargets::default()` — an empty target list. So on an old node a record whose `secretKey` was removed does not fail per target: **every target in that bucket disappears, replication stops, and no caller sees an error.** The raw bytes survive in the blob, so it is recoverable, but the silence is the hazard. Emptying the field instead of removing it avoids triggering it, and the substitution itself should be replaced by a retained parse failure before any of this ships — see [Prerequisites](#prerequisites-in-this-order).
An emptied `secretKey` is not yet a clean local failure either. `build_remote_s3_client` raises `RemoteS3ClientError::MissingCredentials` only when the whole credentials object is absent, and `remote_sdk_credentials` passes an empty secret to the SDK, so today an emptied field signs a request that the remote rejects. That is loud rather than silent, and therefore acceptable as a floor, but the reader-first release should turn an empty access key or secret key into the same typed local error so the failure is attributable to this node instead of to the remote.
**Tiers.** A format change to `tier-config.bin` takes out every tier at once, and tiers are not only a write-path concern: an object already transitioned to a tier cannot be read without that tier's configuration, so the failure reaches GETs of data that has been there for months. The sealed values therefore belong in a companion object under the same prefix, covered by the same `tier_config_lock_path`, keyed by tier name and field name, leaving `tier-config.bin` byte-shaped exactly as it is with an empty `SecretKey`. Putting the envelope *into* `SecretKey` was considered and rejected: an old node would sign requests with the ciphertext, producing remote 403s and ciphertext in signature-related logs, instead of taking its missing-credential path. Confirm the exact decode behaviour against the encode/decode tests in `crates/ecstore/src/services/tier/tier.rs` before writing a byte of the new layout, and do not bump `TIER_CONFIG_VERSION` until every node in the supported upgrade range reads it.
**Downgrade** is the same event as "old node reads new bytes", with one addition: a node that has been downgraded keeps writing the old shape, so a configuration re-submitted through it loses the sealed field and returns to plaintext. That is a security regression, not a correctness one, and it is silent — which is another reason the write side is gated rather than defaulted on.
## KMS unavailable: read time versus write time
These two are not symmetric, and conflating them is how this design would cause an outage.
**At write time** the answer is easy: sealing fails, the admin write is refused with the typed error, and nothing is persisted. A configuration is never stored with the secret dropped, and never stored in clear text after the operator asked for sealing. The cost is that configuration cannot be changed while the KMS is down, which is acceptable and visible.
**At read time** the rule is that a credential which cannot be unsealed makes a remote *unusable*, never *absent*.
- Because unsealing happens at `build_remote_s3_client`, a KMS outage does not change which targets or tiers exist. Listings, status and admin reads keep returning them; each attempt to use one fails with a typed, retryable error that names the KMS as the cause.
- Startup must not treat "cannot unseal" as "no such tier". A tier whose credential is unavailable stays present in `TierConfigMgr`, so a GET of an object transitioned to it fails with a retryable error rather than presenting as missing data, and nothing re-drives a transition elsewhere. The same holds for a replication target: it stays configured and reports why it is not working.
- **A write must refuse to rewrite a configuration it could not fully read.** This is the sharpest edge in the whole design. If a partially-unreadable configuration can be re-serialized from a partially-populated in-memory view, then a KMS outage plus one unrelated admin edit persists the configuration with the unreadable records dropped — and that is the only mechanism by which a target or tier really would disappear for good. Today's code does not have this hazard, because both stores keep raw bytes or fail the whole decode; any per-record sealed handling that skips undecodable records would introduce it.
## Rotation
The envelope records the key id and key version it was wrapped under. Re-wrapping is the KMS side's job and follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md); nothing in this design rotates, re-wraps, or expires a key on its own.
The envelope records the key id and, when the backend reports one, the key version. Re-wrapping is the KMS side's job, follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) and rustfs/backlog#1637 and #1642, and nothing here rotates, re-wraps or expires a key on its own. Two properties make that division workable, and both are constraints on the rewrap job rather than on this design.
- A rewrap must reproduce the encryption context, and the envelope does not carry it. The job must therefore reach a ciphertext **through its store** — enumerate targets, tiers and migration sources and derive the `SealScope` from the record's own position — rather than by scanning for envelope-shaped bytes.
- `key_version` is absent for backends that report none. An absent version means "not known to be current", never "current"; a rewrap sweep must be able to act on it, and a completed sweep is evidence about scanned sources only, exactly as the referenced contract already says about key deletion.
## Fail-closed rules
- A missing hook, a malformed envelope, an unknown envelope version, a failed decrypt, or an encryption-context mismatch is a typed error, per the AGENTS.md rule that required values return a typed error when absent or corrupt.
- A seal failure fails the admin write. A configuration is never persisted with the secret dropped or left in clear text after the operator asked for sealing.
- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed.
- Logs may carry the key id and envelope version. They never carry ciphertext, plaintext, or the encryption context's secret-adjacent values.
- A missing sealer, a malformed envelope, an unknown envelope version, a failed decrypt or an encryption-context mismatch is a typed error, per the root `AGENTS.md` rule that a required value returns a typed error when it is absent or corrupt. `SealedCredentialError` has no variant that degrades to a default, an empty credential, or the raw ciphertext.
- A seal failure fails the admin write.
- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed. `SealedCredential`'s own `Debug` prints the key id and a byte count, not the ciphertext.
- Logs may carry the key id and the envelope version. They never carry ciphertext, plaintext, or an encryption-context value.
- A sealed value never enters an equality or fingerprint comparison. `tier_config_fingerprint` hashes a tier configuration to decide whether an edit is a no-op, and `OnDemandMigrationConfig` derives `PartialEq`; a fresh nonce per seal would make every write look like a change and churn the tier driver cache. Compare plaintext configurations, then seal.
## Alternatives considered
**Do not encrypt; harden the existing boundary instead.** This is the strongest alternative, not a foil. Its parts: keep `.rustfs.sys` off every request surface, which already holds; make an unparseable `bucket-targets.json` fail closed instead of becoming an empty list, which is a bug fix worth doing regardless; prefer keyless credentials where the remote supports them, as `TierS3::aws_role_web_identity_token_file` already allows; and encrypt the drives, which removes the media threat completely, covers all three stores plus every other secret in `.rustfs.sys`, and costs no code. Against the media threat, full-disk encryption strictly dominates application-level sealing. Sealing wins only where the KMS is Transit or AWS **and** the operator wants each unwrap to be individually authenticated, logged and revocable — which is exactly the scope this ADR claims and no more.
**Encrypt the whole blob, as MinIO does for its tier configuration.** Rejected. `tier-config.bin`'s header is what tells a reader the format, and a whole-blob ciphertext makes every tier unreadable whenever the KMS is unreachable; for the bucket metadata blob it would take policy, lifecycle, versioning and object lock down with the credential. Per-field sealing keeps the blast radius at one credential.
**Keep the credential in the KMS and store only a reference.** Rejected. It makes the KMS the durability authority for configuration, adds a second lifecycle with its own orphans when a bucket or tier is deleted, and none of the supported backends is a general secret store — the backends documented in [../operations/kms-backend-security.md](../operations/kms-backend-security.md) manage keys, not arbitrary secrets.
**Deterministic encryption so ciphertext is stable across writes.** Rejected. It weakens the encryption to make ciphertext comparable, and the thing that wanted comparable bytes — configuration-change detection — is correctly solved by comparing plaintext configurations before sealing.
**Seal inside `TierConfig` rather than at the persistence boundary.** Rejected. That type's `Clone` is `redacted()`, so cloning drops secrets, and `tier_config_fingerprint` hashes the type; a nondeterministic sealed field inside it would be both lossy and churn-inducing.
**Encrypt with a node-local key instead of the KMS.** Rejected. The key would sit on the same host as the data, so it removes nothing the reserved path does not already remove, and it creates key material that nothing rotates.
## Prerequisites, in this order
1. Make an unparseable `bucket-targets.json` fail closed in `BucketMetadata::parse_all_configs` instead of substituting `BucketTargets::default()`. This is independently correct and it is what keeps a later mistake from being silent.
2. Make an empty access key or secret key a typed `RemoteS3ClientError` in `remote_sdk_credentials`, so an emptied plaintext field fails on this node rather than as a signature rejection at the remote.
3. Ship a reader-first release: every store tolerates the sealed field and the emptied plaintext field, and nothing writes either. For on-demand migration this means relaxing `deny_unknown_fields` for exactly that field name; for tiers it means reading the companion object when present.
4. Only then enable writing, gated on KMS being configured and on a module switch in `rustfs/src/module_switches.rs` that defaults off in the release introducing it. Operators turn it on once every node reads the format. Existing plaintext records convert by re-submitting the configuration through its admin API; this work ships no in-place migration sweep.
Steps 1 through 3 each introduce a compatibility path that needs a `RUSTFS_COMPAT_TODO` marker and a matching entry in [compat-cleanup-register.md](compat-cleanup-register.md) when the code lands. This document adds neither, because the guard matches markers and register entries in both directions and an entry without a marker fails it.
## Non-goals
Sealing the server config, IAM credentials, or object data keys; changing which principals may read a configuration; key material migration between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the existing trust boundary (reserved bucket paths plus admin authorization) is unchanged.
Sealing the server configuration, IAM credentials or object data keys; changing which principals may read a configuration; migrating key material between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the boundary described above is unchanged.
docs/architecture/unified-object-generation.md
+146 -81
View File
@@ -1,112 +1,177 @@
# Object Transaction UUID And Generation-Fencing Contract
# Object Generation Authority And Recovery Contract
**Use this when:** adding or changing anything that fences a commit, scopes a read lease, gates old-directory cleanup, binds prepared pool reads, or settles quota against "the current version of an object", or when adding a field that rides internode RPC or `xl.meta`.
**Source of truth:** `assign_object_transaction_epoch` in `crates/ecstore/src/set_disk/ops/object.rs` and `crates/ecstore/src/set_disk/ops/multipart.rs`; `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs`; `commit_rename_data_dir` and `RenameConvergence` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `PreparedPoolReadFallbackBarrier` in `crates/ecstore/src/store/rebalance.rs`; `crates/protos/src/node.proto`; env constants in `crates/config/src/constants/object.rs` and `crates/config/src/constants/internode.rs`.
**Use this when:** changing object commit fencing, rollback, old-directory cleanup, prepared reads, quota settlement, or the metadata and RPC fields used by those operations.
**Source of truth:** `crates/ecstore/src/set_disk/ops/object.rs` (`assign_object_transaction_epoch`, `verify_object_transaction_epoch_fence`); `crates/ecstore/src/set_disk/core/io_primitives.rs` (`rename_data_owned_with_fence`, `commit_rename_data_dir`); `crates/ecstore/src/disk/local.rs` (`rename_data`, `write_all_meta`); `crates/lock/src/distributed_lock.rs` (`DistributedLockGuard`, `LockLostSignal`). The implementation boundary below distinguishes existing behavior from the selected design.
Design tracking lives in `rustfs/backlog#1326`. This document holds only the invariants.
## Decision And Implementation Boundary
## Authority
The selected minimum authority is a **durable, ordered per-object decision protocol attached to the existing namespace-lock participant group**. The object transaction UUID remains an opaque operation/idempotency identifier. It is not an ordered lock epoch. Extending the existing lock group requires durable promises, accepted values, quorum decisions, and recovery; adding a counter to today's lock response is insufficient.
The target contract requires **one per-object commit identity** consumed by commit fencing, read leases, cleanup, prepared reads, and quota settlement. No consumer may mint a second value and call it the same generation.
An independent service holding every object's full manifest is not selected. It would add a new routing, membership, availability, and metadata ownership system and require a wider read/write migration. The selected protocol stores the current decision and recoverable outstanding successor with the existing lock participants; object payload and prepared metadata remain on the existing storage disks. This is still new consensus and persistence work, not a small `RenameData` patch.
What exists today is an **object transaction UUID**, not the target authority:
**Implementation status:** this document does not implement or claim distributed generation authority. Existing fencing remains an opt-in coordinator equality recheck. `rustfs/backlog#2251` cannot be completed by forwarding the UUID to disks and adding local CAS. Its implementation must be split at the protocol boundaries in [Required Implementation Boundaries](#required-implementation-boundaries), with the availability and rollout changes reviewed before strict activation. The two original requirements “commit with a quorum while a disk is unreachable” and “every disk immediately rejects every older request” cannot both hold; the precise target below preserves quorum availability.
| Property | Current implementation |
Related contracts remain authoritative for their domains: [erasure-coding.md](erasure-coding.md) defines data durability and voting, [heal-concurrency-model.md](heal-concurrency-model.md) defines namespace-lock scope, [placement-repair-invariants.md](placement-repair-invariants.md) defines placement and repair admission, and [minio-file-format-compat.md](minio-file-format-compat.md) defines format interoperability.
## Current Guarantee And Counterexamples
`assign_object_transaction_epoch` generates a random UUID for gated PUT and CompleteMultipartUpload. `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs` stores it under both internal metadata prefixes. `verify_object_transaction_epoch_fence` re-reads quorum metadata before the rename fanout, outside the eventual per-disk mutation critical section. `ObjectTransactionEpochFence::Absent` currently covers both an absent object and existing metadata without a UUID. Cleanup receipts compare UUID equality. None of these operations is a durable distributed CAS.
The lock implementation already bounds lease validity. `LockLostSignal::is_lost` includes the conservative deadline; `DistributedLockGuard::run_heartbeat` retains prior deadlines after transient RPC failure and reports loss when refresh quorum is no longer valid. `LocalClient` in `crates/lock/src/client/local.rs` keeps `LocalGuardEntry` in an in-memory map; `LockResponse` in `crates/lock/src/types.rs` contains no persisted ballot or accepted object decision. Restarting a lock participant therefore cannot supply the durable order required here.
These schedules disprove a local UUID-CAS replacement, even assuming a perfect local mutex and atomic metadata replacement. They are protocol counterexamples, not claims that a multi-node fault test has already been run.
| Schedule | Result and implication |
|---|---|
| Minting | `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and CompleteMultipartUpload when the object-transaction gate is active. |
| Persistence | Written through `FileInfo::set_object_transaction_epoch` into the version's internal metadata map under the dual-key contract (`x-rustfs-internal-*` / `x-minio-internal-*`). |
| Fence check | The coordinator reads the current UUID (or `Absent`) and revalidates exact equality immediately before `rename_data`. |
| Cleanup | Old-data cleanup receipts carry the committed UUID; reconciliation deletes only when the receipt UUID still equals the current object UUID. |
| Four disks start at X; A captures expected X and stalls. B commits B on d1–d3, satisfying W=3. d4 has not heard from B. A reaches d4 with expected X. | d4's exact-CAS accepts A. A durable local `highest_ballot` also accepts if d4 never received B's ballot. Neither mechanism proves rejection on every disk after a quorum commit. |
| A changes d1,d2 from X to A; after lock loss B changes d3,d4 from X to B. Each refuses the other's disks because expected X no longer matches. | Neither reaches W=3, even with all disks now reachable. Equality-CAS alone has no rule for choosing recovery, retaining uncertain work, or safely retiring it. This is a recovery/liveness counterexample, not proof of two successful intersecting write quorums. |
| A reaches W=3; its reply is lost. A's coordinator restarts and sees a partial or changed disk view. A's rollback runs after B has replaced A. | Timeout is not evidence of abort. Restoring A's backup can erase B unless rollback names its own committed effect and consults a durable decision. |
| A checks a lease/UUID; B later commits; A's blocking syscall resumes. | Another coordinator-side check, cancellation token, or process-local mutex cannot establish an atomic cross-node order. The mutation itself must consume the protocol state. |
This is an equality-CAS fence and cleanup identity. It is not a monotonic epoch, is not minted by the distributed lock grant, and is not compared atomically at each disk's `xl.meta` commit point. Documents and issues must call it the *object transaction UUID*, not proof that the generation authority exists.
A lower-ballot write on an isolated stale disk cannot become the authoritative object. A disk that has applied B must never replace B with A. An uncontacted disk may retain older committed materialization until recovery; it must not vote that state as a newer decision or authorize cleanup. Requiring all disks to learn B before acknowledging it would change W to N or require successful isolation of every unreachable disk. That availability change is rejected for the selected design and must not be hidden inside E03's tests.
### Authority modes (one must be selected)
## Authority, Identity, And State
| Mode | Contract | Persistence requirement |
|---|---|---|
| Total-ordered fencing epoch | A lock grant returns a durable per-object `(term, counter)`; every disk rejects a lower epoch at the atomic metadata commit point; the value never regresses across lock-plane restart, failover, or minority recovery. | Quorum-persisted before grant, or derived from a durable term whose full comparison cannot regress. The in-memory distributed lock entry alone is insufficient. |
| Opaque commit-generation identity | Consumers compare exact identity only; no `<` / `>` semantics. The authoritative commit performs an atomic expected-generation CAS; lease, cleanup, prepared-read, and quota contracts are phrased as "references this exact generation". | Atomic expected-identity comparison plus durable crash recovery. |
The authority key is `(bucket incarnation, bucket, object key)`, covering the whole object version set. It is not the S3 version ID. Deleting a noncurrent version, updating tags, or recording replication status can change the authoritative revision while the current S3 version remains the same. The bucket incarnation prevents reuse after bucket deletion/recreation.
The current UUID proves neither a durable total order nor a per-disk atomic CAS, so it does not decide between the modes.
The protocol has separate typed values:
## Consumer Binding
- `Ballot = (configuration epoch, counter, durable proposer ID)`, compared lexicographically only within the specified authority configuration. The proposer persists a counter before use and raises it above every observed promise. Restart never resets it; exhaustion is an error. A different process boot gets a new transport epoch, not permission to reuse a ballot for different bytes.
- `Generation = (object revision, operation UUID)`, allocated by a chosen successor decision. Revisions increase from the committed predecessor; UUIDs are compared for equality only. Never infer generation order from modification time, version ID, or UUID bytes.
- `DecisionValue = (authority key, predecessor generation, successor generation, operation kind, semantic metadata digest, per-disk prepared metadata/data receipts, outcome identity)`. Disk-specific erasure indices, checksums, and metadata blobs are bound by individual receipts, not assumed byte-identical across disks. The semantic digest includes the full version set and relevant metadata, including fields omitted from ordinary read voting.
| Consumer | Binds generation how | Key invariant | Current state |
|---|---|---|---|
| Commit fence (PUT / CompleteMultipartUpload) | Checked at `rename`, rollback restore/delete, and cleanup mutation points using the selected rule | A stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | Opt-in UUID equality recheck before rename; no per-disk atomic comparison |
| Read lease | Lease binds the exact generation observed at read time; GC runs only after every lease on that generation is released | Lease visible across nodes; crashed reader's lease reclaimed by TTL | Streaming/multipart GET holds the namespace read lock through EOF/drop (part-boundary coverage: `#6887`); no cross-node generation-bound registry |
| Old-dir GC | Cleanup job carries the committed generation and confirms no lease owns `old_dir` before deleting | `old_dir != committed_dir`; a still-referenced directory is never deleted | UUID receipt equality (`#6077`); no lease consultation |
| Prepared pool read | The prepared bundle carries the generation resolved during pool lookup; the chosen pool reuses it only after a match | Mismatch forces fallback to full metadata fanout | `PreparedPoolReadFallbackBarrier` (`#6889`) is a pool-local identity that fails closed / refetches on pool state change; it is not a cross-pool authority |
| Quota reservation | Reserve / settle record binds the exact object generation (and the ordered epoch too, if selected) | A late commit cannot settle quota for a different committed generation | Durable per-bucket ledger with independent snapshot-lease fence tokens (`#6058`); not bound to the transaction UUID |
A decision does not require a new `FileInfo` positional field. The operation UUID stays in the existing metadata map. Durable authority records and local recovery records carry the revision, predecessor, ballot, and complete decision identity. They use separately versioned records; they cannot be inferred from a version's UUID alone. `xl.meta` remains a recoverable materialization of the chosen decision in strict mode.
## Fence Coverage: Three Disk-Write Points
Checking generation only before the `rename` fanout is insufficient. The commit sequence is `tmp sync → data-dir rename → xl.meta commit → directory sync` in `crates/ecstore/src/disk/local.rs`, and `crates/ecstore/src/set_disk/core/io_primitives.rs` has two further detachable disk-write points:
1. **Rollback restore/delete.** On quorum failure each disk can restore backup metadata or delete the failed version. A stale writer's rollback must compare the expected generation, or it can overwrite or delete the winner's committed metadata. Panic, cancel, and timeout outcomes must be reaped into coordinator convergence rather than skip rollback through an early return.
2. **`commit_rename_data_dir`.** A cancel-then-detach disk-write point; the coordinator's "reap all child tasks" must include it so a cancelled writer cannot bypass fence or lease and keep deleting directories.
If generation is validated only after the data-dir rename, a fenced writer may already have renamed its data-dir into the object path, leaving a staged orphan. Either move the fence ahead of the data-dir rename, or declare that orphan an accepted residue accounted for by GC metrics.
`RenameConvergence` (`AllSuccessIdentical` / `PartialCommit` / `SignatureDivergent` / `Unknown`) is a *post-commit* heal signal on the same `rename_data` path; the fence is a *commit* gate. They compose: the fence decides whether a convergence is produced, `RenameConvergence` classifies it. A fence-aware convergence variant would be an additive enum change.
## Transport And Security
Generation and derived tokens (lease, reservation) cross node boundaries in internode RPC bodies; every such flow must be signature-bound.
| Rule | Detail |
| Input/state | Required treatment |
|---|---|
| HMAC scope | Target audience, exact service/method, timestamp, nonce, canonical body digest, receiver replay (boot) epoch. The receiver consumes the nonce in a bounded replay cache; a transmitted-but-unconsumed nonce is not replay protection. |
| Current substrate | RPC v2/v3 in `crates/ecstore/src/cluster/rpc/http_auth.rs` binds all of the above. Body-bound policy covers mutating disk RPCs including `RenameData`, whose versioned canonical body includes every `RenameDataRequest` field, so the `FileInfo` metadata map carrying the UUID is authenticated. |
| Strict switches | `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` (`crates/config/src/constants/internode.rs`) are default-off rollout gates governed by [compat-cleanup-register.md](compat-cleanup-register.md). A generation capability may claim strong transport binding only after the relevant strict modes have converged fleet-wide. |
| Acceptance tests per consumer | Method substitution, canonical body tamper, nonce replay, receiver restart, stripped-strict-metadata negatives. |
| Object never present | An explicit absent bootstrap state, established during strict cutover; no client-supplied `None` may authorize creation. |
| Object deleted, including deletion of its last version | A durable tombstone head with its own revision; never revert to never-present. This prevents ABA and resurrection after a laggard rejoins. |
| Null version | A real version-set member, distinct from absent. Replacing/removing it changes the object head. |
| Delete marker | A real version-set member and, when latest, a deleted-current state. Preserve marker type and existing S3 behavior. |
| Noncurrent version mutation | Compare the whole-object predecessor and the selected version's identity; commit a successor even if the latest S3 version is unchanged. |
| Valid legacy metadata with no UUID | Import once during fenced cutover after ordinary quorum/format validation; assign a bootstrap generation in the authority. Never equate this with absent or silently import while old writers are still admitted. |
| Missing, malformed, nil, or conflicting dual generation keys after strict enrollment | Typed corrupt/unsupported-state error; retain data for repair. No default to zero, absent, or a new UUID. |
| Missing/lagging/replacement disk | A non-authoritative materialization target. Recover the chosen decision and validate its data before it can contribute; do not demand that it already equals the predecessor or blindly overwrite it. |
## Encoding Rules
## Durable Decision Protocol
| Rule | Reason |
Use the existing namespace-lock participant identities and routing, with an explicitly persisted authority configuration. Lock voters and erasure disks are different sets: `Qlock = floor(lock_participants / 2) + 1` decides authority; the existing per-operation `Wdata` decides recoverable object durability. A lock vote is not a shard receipt. One-node deployments still persist their single voter's state. Membership changes cannot be inferred from whichever RPC endpoints answered.
Each participant persists, per authority key and successor slot, its highest promise, highest accepted `(ballot, DecisionValue)`, and the last learned committed head. Promises and accepted records survive unlock, TTL expiry, restart, and log compaction. Durable records are written through a storage boundary below the object API; writing them through PUT would recursively acquire the same authority. `LocalClient` must not become a filesystem implementation: the lock crate consumes an injected durability interface, while the storage owner implements it.
The following is a protocol contract, not pseudocode to paste into the current rename implementation:
1. Acquire the existing object namespace write lock for admission. Read/recover the latest authority head through a fresh quorum promise/recovery barrier; reading only cached learned-head markers is insufficient because a quorum may have accepted a value before its commit notification arrived. Resolve any accepted successor before returning a head or allocating another slot. An unavailable decision quorum is an explicit failure. The lock still prevents ordinary competing work, but it is not the safety proof after lease loss.
2. Stage the new shards and exact replacement metadata in transaction-owned paths. Obtain Wdata receipts only after the required file and directory syncs. Receipts bind disk identity/incarnation, authority configuration, key, operation UUID, blob digests, and data directories. Preparing must not replace live metadata, remove old directories, or reuse a winner's directory. Metadata-only/delete operations also stage a recoverable replacement version set.
3. For successor slot `head.revision + 1`, obtain Qlock durable promises for a unique ballot. Each response returns its accepted value, if any. Adopt the value with the highest accepted ballot among the promise quorum. Only if none was accepted may the proposer offer its own candidate with that predecessor. A recovered candidate is not replaced merely because its coordinator timed out or its lease expired.
4. Validate the candidate's Wdata preparation receipts and predecessor, then obtain Qlock durable accepts of the **same** value at that ballot. A participant accepts only at or above its promise and must reject a different value at the same ballot. A value becomes chosen at Qlock acceptance. Quorum intersection plus adoption of the highest accepted value prevents a different value from being chosen in that slot. Retrying the same operation cannot allocate a second successor.
5. Learn/persist the chosen decision and publish it on storage disks through the guarded local recovery protocol below. Return S3 success only after both a durable chosen decision and Wdata durable materializations satisfy the existing operation's rules. Decision chosen but publication incomplete is `OutcomeUnknown/PendingRecovery`, never authorization to roll back the decision. A later proposer first resolves the prior slot before allocating the next one.
6. On a lost ACK, resolve by operation UUID and exact request digest. The results are `NotChosen`, `ChosenPendingPublication`, `Committed`, or `SupersededAfterCommit`. Reusing a UUID for different input is invalid. A timeout without a recovered decision stays unknown. Record idempotency outcomes until the client retry horizon and all dependent cleanup/accounting records have passed a durable retirement watermark; old requests then fail as expired rather than being treated as new.
The decision and payload retention lifetimes are coupled. Accepted/staged data is not garbage merely because no coordinator is alive. A new promise quorum can adopt a previously accepted value and finish it. If the required payload has been physically lost, fail closed and repair; never choose a different value for an already chosen slot. Persisted voter state that is lost or corrupt requires catch-up/replacement, not an empty voter with the same identity.
This is the minimum extension that turns the lock grant into a recoverable authority. A promise-only grant lacks outcome recovery; per-disk promises without a common decision allow minority uncertainty to escape into reads. The protocol requires review and executable state-machine tests before production integration. It does not imply that the current lock RPC is already a consensus implementation.
## Single-Disk Publication And Recovery
Local and remote disks execute the same guarded primitive. A remote RPC handler must decode and authenticate the request, then call that primitive; checking only in the RPC handler leaves local callers and deferred syscalls uncovered. All operations touching the object's metadata, backup, or referenced directories participate.
Under one object mutation guard, re-read local durable recovery state, verify the decision/configuration and local nonregression condition, record a write-ahead intent, sync it, perform data-directory rename and atomic metadata replacement, sync affected directories, then persist applied outcome. The guard, including ownership of any namespace/deletion lease, stays with the blocking syscall until it completes, even if its caller is cancelled. An async task disappearing must not release a guard while its syscall still runs.
A single filesystem rename does not atomically commit a sidecar plus `xl.meta`. The write-ahead record binds predecessor/successor identities and exact metadata bytes; startup recovery runs before disk readiness. Recovery replays a chosen intent forward and completes syncs. An unchosen staged operation stays private until authority recovery makes its retirement safe. Old snapshots never overwrite a newer applied local revision. Conflicting bytes for the same decision are corruption. Treat write/fsync errors and torn records as unknown until decoded and reconciled, not as successful rollback.
A laggard need not contain the predecessor. Recovery fetches the chosen decision and its verified metadata, reconstructs or validates its shards under [erasure-coding.md](erasure-coding.md), and installs that state. An empty replacement uses a fresh disk incarnation and cannot reuse old preparation receipts. A disk whose durable state claims a later decision than the supplied one rejects the operation; a conflicting same-revision digest is quarantined. Never erase a divergent disk merely because it is in the minority.
`rollback_committed_rename_std`, `rollback_inline_metadata_commit_std`, and `restore_metadata_backup` in `crates/ecstore/src/disk/local.rs` must become decision-aware before strict mode includes them. The permitted rollback is limited to a transaction's unchosen private preparation, or restoration proven by recovery to be necessary before any newer local effect. A chosen operation is repaired forward. Neither a client timeout nor a rename-tail error permits reverting an acknowledged decision.
`RenameConvergence` remains a post-publication repair signal. `PartialCommit`, `SignatureDivergent`, and `Unknown` do not decide which transaction won. Keep their diagnostics and quorum accounting; resolve authority first. Early ACK may still precede minority-tail completion after the two quorum conditions hold. Tests that inspect all disks must synchronize the tail or assert the permitted minority residue separately.
## Writer Participation
Every semantic metadata change advances the whole-object generation, including metadata-only writes. A physical repair that reproduces exactly the already chosen bytes preserves the generation and consumes that chosen decision; it must not create a new semantic value. The table specifies participation, not a generated inventory of every call site.
| Writer and current code boundary | Required generation behavior |
|---|---|
| **Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION`** (`crates/filemeta/src/filemeta.rs`). | `decode_xl_headers` in `crates/filemeta/src/filemeta/codec.rs` rejects newer values outright; a bump makes every new `xl.meta` unreadable by rolling-upgrade old nodes and by MinIO. See [minio-file-format-compat.md](minio-file-format-compat.md). |
| **Do not add generation as a `FileInfo` struct field.** | Internode RPC serializes `FileInfo` with two msgpack encoders: positional-array encoding for the `read_version` family (a new positional field breaks mixed-version decode) and `encode_msgpack_named` (named-map) for `rename_data` in `rustfs/src/storage/rpc/node_service/disk.rs`. A field would have to be correct under both plus the JSON compatibility twin. Use the metadata map, which rides every encoder unchanged. |
| **Metadata-map dual key.** | The UUID lives under `x-rustfs-internal-*` / `x-minio-internal-*`; missing, malformed, nil, or conflicting dual values fail closed when fencing is active. |
| **No sidecar unless atomic.** | An epoch sidecar outside `xl.meta` is admissible only if it commits at the same atomic/CAS point as `xl.meta` with a specified crash-recovery protocol. None is implemented. |
| **Regression guard.** | The real-MinIO `xl.meta` interop fixtures in `crates/filemeta/src/filemeta.rs` must keep passing: objects written by a new node stay readable by old RustFS nodes and by MinIO in both upgrade directions. |
| PUT / data COPY: `put_object_with_old_current_size_inner`, `copy_object` in `crates/ecstore/src/set_disk/ops/object.rs` | Stage, choose, publish a successor; preserve source read protection. A metadata-only COPY is also a semantic successor, even if data directories are shared. |
| MPU: `complete_multipart_upload`, `new_multipart_upload`, `abort_multipart_upload` in `crates/ecstore/src/set_disk/ops/multipart.rs` | Complete chooses the destination object's successor. Part staging/upload metadata and abort remain in the upload namespace; they cannot delete a directory transferred to a chosen object decision. |
| DELETE, batch DELETE, null/marker removal: `delete_object`, `delete_objects_with_accounting`, `delete_object_version` in `crates/ecstore/src/set_disk/ops/object.rs`; lifecycle callers in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | Each object has its own predecessor/decision; retain tombstone authority after the last version. Prefix deletion must enumerate decisions or prove a bucket-incarnation retirement barrier; a recursive bypass is forbidden in strict mode. |
| Heal: `heal_object_with_explicit_version_regen` in `crates/ecstore/src/set_disk/ops/heal.rs` | Exact repair preserves the chosen generation and verifies full metadata identity. A version-list or semantic metadata change needs a successor. `no_lock` may skip admission only; it cannot bypass authority. In-place directory repair cannot remove a reader's live directory. |
| Transition / restore: `transition_object`, `restore_transitioned_object`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`; `finalize_restore_metadata`, `update_restore_metadata` in `crates/ecstore/src/set_disk/replication.rs` | Each metadata transition is a successor, preserving existing operation-ID, remote tuple, and tier lease checks. Bind the transition transaction to the exact predecessor/successor; a late finalizer cannot rebase onto another restore operation. |
| Replication status and metadata/tag/retention writeback: `put_object_metadata`, `put_object_tags`, `delete_object_tags`, `merge_replication_metadata_lww` in `crates/ecstore/src/set_disk/ops/object.rs`; callers in `crates/ecstore/src/bucket/replication/replication_resyncer.rs` | Commit a field-scoped successor conditional on the exact version/content identity. On conflict, reload and revalidate the mutation; never replay a full stale `FileInfo`. Existing LWW category rules remain applicable within that validation. |
| Rebalance/decommission: `migrate_entry_version` in `crates/ecstore/src/services/rebalance/migration.rs`; `decommission_tier_free_version`, `decommission_tiered_object` in `crates/ecstore/src/set_disk/mod.rs`; `crates/ecstore/src/data_movement/mod.rs` | The authority key and lock group remain stable across pools. Stage the destination, choose the location/ownership successor, then retire the exact source receipt. Do not mint independent source and destination authorities. Existing placement and tier ownership fences remain required. |
| Generic metadata entry points: `write_unique_file_info`, `update_object_meta_with_opts` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `LocalDisk::write_metadata`, `update_metadata`, `delete_version`, `delete_versions_internal`, `write_all_meta` in `crates/ecstore/src/disk/local.rs` | Consume a validated decision/recovery context or reject strict writes to enrolled objects. None may invent a generation, reset it through `fresh`, replace corrupt metadata with an unproven empty version set, or bypass durability with `no_persistence`. |
| Rollback and GC: `rename_data_owned_with_fence`, `commit_rename_data_dir`, `reclaim_orphan_data_dirs` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `reconcile_old_data_cleanup_receipts` in `crates/ecstore/src/set_disk/ops/object.rs` | Consume the owning decision and exact directory references. Cleanup does not change object contents or grant a new semantic generation. It must use a durable retirement decision and local reader/deletion guards. |
### Wire-encoding window (JSON and msgpack)
Strict capability is withheld until every raw writer in `DiskAPI`, its local implementation, `DiskStore`, remote adapters, and server handlers has an enforced path. Internal authority persistence must use its own narrow storage primitive, not evade this rule by recursively calling generic object metadata writes.
- Dual-encoded RPC fields exist twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes *_bin` field (e.g. `file_info` and `file_info_bin` on `RenameDataRequest`). Senders emit both; receivers (`decode_msgpack_or_json` in `crates/ecstore/src/cluster/rpc/remote_disk.rs`) prefer `_bin` and fall back to JSON only when `_bin` is empty.
- `rustfs_protos::internode_rpc_msgpack_only()` drops the JSON copy only when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` are set after the JSON-fallback metric reads zero fleet-wide.
- Generation inside the `FileInfo` metadata map is carried in both copies automatically. Any new *top-level* generation datum must be added to both encodings and be safe under both msgpack encoders; a field in only one encoding is silently lost when a peer falls back.
- `RenameDataRequest` has a versioned, injective canonical-body encoder covering both compatibility fields; a strict generation-capable request must reject missing or mismatched canonical-body metadata rather than downgrade to the unauthenticated JSON twin.
## Reads, Garbage Collection, And Accounting
### Proto evolution
Strict reads need the chosen head, not a majority of arbitrary prepared/live UUIDs. Resolve the decision under the namespace read lock before accepting object metadata; validate the selected current or explicit version against it, and wait for or repair missing materialization. HEAD, GET, ListObjects/ListObjectVersions, scanner reads used for deletion, and prepared pool reads all need this distinction. A query may return an error while a chosen write is recovering; it must not expose an unchosen candidate or resurrect a retired version. This read-decision adapter is part of the strict-mode scope and is a reason E03 is larger than disk CAS.
No top-level proto field is required by the metadata-map UUID. If an ordered epoch or explicit expected-generation is ever added to proto, it uses **proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: an old coordinator talking to a new disk decodes absence as a plausible zero.
Keep the current namespace read lock through EOF/drop, including multipart part boundaries. This design does not replace it with a new cross-node generation lease registry. A strict implementation must also bind every deferred local/remote part open to the resolved generation and acquire protection on the disk that owns the directory before handing the read capability out. A reader that loses authority or cannot renew its disk protection must fail before another open; it cannot continue on an unvalidated cached pathname.
## Mixed-Version Gate: One Direction
`LocalDisk::acquire_snapshot_lease`, `renew_snapshot_lease`, `release_snapshot_lease`, and `delete_data_dir` in `crates/ecstore/src/disk/local.rs` provide disk-local path protection and deletion deferral. They are not proof of a fleet-wide object generation. Before reuse, their token must bind disk incarnation and exact generation/directory, and strict reads must reject a pre-restart token. Existing open file descriptors may finish reading an unlinked inode, but later part opens need a valid protected generation. This preserves streaming behavior without assuming a local mutex protects another node.
When generation enforcement is not explicitly requested, or fleet confirmation is absent, behavior falls back to current semantics. Fail-closed is reserved for an explicit administrator-confirmed strict rollout.
GC consumes a durable retirement authorization for exact directories no longer referenced by **any** retained version or pending accepted decision. Include retirement in the chosen successor that removes the last reference; if it was not recorded there, choose a metadata-neutral successor that records it before deletion. That successor advances the authority revision while preserving the S3 version contents; the cleanup syscall itself never mints an identity. Retirement prevents future repairs/reads from creating new references; a new reference requires a new decision and cannot revive a retired directory. At the destructive syscall, hold the object/directory guard, recheck the local chosen metadata references, `old_dir != committed_dir`, retirement/configuration identity, and local snapshot protection. Any uncertainty defers deletion. A stale cleanup receipt matching an earlier UUID is not enough. Across executor restart, replay the same retirement ID idempotently; do not convert a lost reply into a broader recursive delete.
| Flag (`crates/config/src/constants/object.rs`) | Default | Effect |
|---|---|---|
| `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` | false | With either flag absent, PUT/MPU neither persists nor consumes the transaction UUID. |
| `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` | false | With both enabled, failure to obtain or retain the live fleet proof rejects the commit before rename. |
Prepared pool reads remain a separate optimization domain. In `crates/ecstore/src/store/rebalance.rs`, `prepare_latest_object_metadata_with_idx` collects candidates and revalidates a refetched winner with `validate_prepared_pool_refetch_identity` from `crates/ecstore/src/store/rebalance/support.rs`. `PreparedPoolReadFallbackBarrier` is a `#[cfg(test)]` scheduling fixture, not a production identity. Keep the all-pool resolution rules until a chosen location decision supplies equivalent evidence. The prepared bundle binds authority generation plus pool identity; mismatch requires a complete refetch or a typed failure, never selection of a different generation using the old bundle.
The fleet proof is currently borrowed from the remote-version-state writer rollout. It proves membership/process-epoch convergence for that feature only; it does not prove an epoch type, per-disk CAS support, or RPC strict-mode convergence, and must not be treated as the final generation handshake.
Quota remains a separate per-bucket arbitration domain. `QuotaLedger` and `settle` in `crates/ecstore/src/bucket/quota/reservation.rs` key reservations by operation UUID, validate object/size, and update under the ledger fence. A late settlement cannot remove a different reservation key; an absent/mismatched key fails or follows the existing idempotent abort rule. That proves key isolation, not that current generation A committed or that a ledger storage write is immune to stale-disk mutation.
### Capability negotiation (target)
For strict mode, add the decision identity to the reservation/settlement binding. Settle a committed **historical** decision even if it has since been superseded, but only against its original reservation and recorded old/new sizes; demanding that it still be current would leak valid reservations. Abort only a recovered unchosen/retired operation. Unknown outcomes stay reserved and reconcile. Retain the existing conservative usage floor and `commit_started` recovery behavior. Ledger writes themselves require the authority protocol, with a documented lock order and no recursion through their own reservation path. No quota token is compared numerically to an object ballot or used to revoke a newer object's leases.
Generation enforcement requires one **live fleet proof** containing at least: the selected authority version and comparison mode; the current membership/topology fingerprint and process epochs; support for every required disk mutation point; RPC signature/body/replay strict convergence; and the on-disk encoding version (the metadata-map UUID is version 1). Membership change or an old-node rejoin revokes the proof; revocation before commit fails an explicitly strict request and never rewrites or lowers a persisted generation. The proof may extend the authenticated fleet-proof machinery in `notification_sys` or the runtime capability contract; this document requires one shared token, not a mechanism.
## Restart, Membership, And Rollout
## Open Decisions
Authority configuration is durable and includes participant identities, routing, bucket incarnation, protocol version, and quorum rules. Changing storage pool placement must not remap the authority key. A replaced voter starts as a non-voter, catches up durable promise/accepted/chosen state, and only joins through a quorum-approved configuration transition. Configuration change requires intersecting old/new decision quorums; losing the old quorum is a recovery incident, not permission to bootstrap a new empty authority. Offline data disks rejoin through generation-aware catch-up, independent of voter admission.
Blockers for calling the contract implemented:
A live capability proof must bind the authority configuration, topology, every participant process boot epoch, disk incarnations, writer/read/recovery protocol support, encoding version, and RPC signature/body/replay strictness. Restart, membership change, disk replacement, protocol downgrade, or a strict-transport setting change revokes it. Receivers revalidate before entering the publication critical section; accepted durable decisions survive proof revocation and are recovered under a fresh valid proof, never replayed as unvalidated requests. The remote-version-state fleet proof does not prove any of these generation capabilities.
1. **Authority mode.** Total order or opaque exact-CAS. Do not retrofit ordering semantics onto the existing random UUID.
2. **Complete `xl.meta`-writer coverage.** Enumerate commit rename, rollback restore/delete, cleanup, heal, transition, restore, replication, and data movement; each path compares/carries the selected generation or is proved incapable of replacing the authoritative identity.
3. **Rollback as expected-generation CAS.** The quorum-failure rollback in `rename_data` restores backup metadata, not just a private temp file; it must run only when the stored generation still matches the failed writer's expectation.
4. **Generation capability proof.** Extend the fleet proof or the runtime capability contract; one revalidatable token.
5. **Read-lease and GC crash recovery.** Cross-node registry, TTL reclamation, lease-holder crash behavior, GC-executor recovery.
6. **Quota reserve → commit → settle binding.** Relate the ledger's independent mutation tokens to the selected generation, with a concrete late-settle rejection test, or prove the fence is a separate arbitration domain that cannot cross-settle.
7. **Prepared reads stay pool-local.** `PreparedPoolReadFallbackBarrier` validates freshness only within the pool that produced it; cross-pool ordering requires a common authority, and the multi-pool wait cannot be short-circuited without one.
8. **Hot-path cost is a blocking metric.** Measure any added consensus write, fsync, fleet-proof lookup, lease operation, or centralized serialization under 4 KiB and hot-key/hot-bucket A/B.
9. **Test infrastructure.** Multi-node, multi-pool, directed network-fault, and large-object budget for restart, mixed-version, and cross-node lease acceptance.
The existing `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` and `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` flags in `crates/config/src/constants/object.rs` retain their current default-off behavior. They do not become a claim that the new protocol exists. If generation strictness is explicitly selected, missing capability is an error, never silent downgrade. No new environment variable is introduced by this document; a production gate must be documented with its implementation.
Strict enrollment requires quiescing old writers and readers for the enrolled namespace, recovering ambiguous operations, validating/importing legacy heads, persisting a strict-format/protocol marker, and enabling the complete fleet. New disks reject unbound legacy mutation RPCs for that namespace. Old binaries must be prevented from opening a strict-enrolled drive by a startup compatibility gate they understand before enrollment; an environment flag known only to new binaries is insufficient. Until that prerequisite is deployed, do not activate strict mode in a mixed fleet. Disabling flags after enrollment cannot drop durable authority; downgrade requires a separately verified quiescent materialization/export operation. Ordinary un-enrolled compatibility deployments keep their current behavior.
## Encoding And Transport
- Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION` in `crates/filemeta/src/filemeta.rs`. Do not add fields to positional-msgpack `FileInfo`; carry the UUID through the metadata map and protocol records through explicit versioned envelopes.
- Write RustFS/MinIO internal metadata dual keys using `crates/utils/src/http/metadata_compat.rs`. Reject conflicting, nil, or malformed generation values; validate every persisted/RPC record again at consumption.
- New proto values in `crates/protos/src/node.proto` require explicit presence (`optional` scalars or a present message), including absent/tombstone state. Bind expected/new generation, ballot, receipts, configuration, and outcome identity in the canonical body. The JSON and msgpack representations must carry identical semantics; absent data from an old peer cannot decode as a valid zero ballot.
- `crates/ecstore/src/cluster/rpc/remote_disk.rs` and `rustfs/src/storage/rpc/node_service/disk.rs` must share local protocol behavior. Extend canonical encoders for every affected mutation, not only `RenameData`. Authenticate both compatibility representations and reject disagreement rather than falling back to a weaker JSON twin.
- `crates/ecstore/src/cluster/rpc/http_auth.rs` supplies signature, canonical-body, and replay-scope checks. Strict generation capability requires fleet convergence of `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, and `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` from `crates/config/src/constants/internode.rs`. Cover method substitution, body tamper, stripped metadata, consumed nonce replay, and receiver restart.
- Preserve real-MinIO metadata fixture decoding and supported old-RustFS compatibility before strict enrollment. Do not promise that a live MinIO binary can start a RustFS-written drive set; [minio-file-format-compat.md](minio-file-format-compat.md) explicitly excludes that direction. Strict authority records also impose a new deployment boundary even though the `xl.meta` container version is unchanged.
## Required Failure Outcomes
Every scenario must first prove its intended barrier, quorum, or crash point was reached, then inspect authority records, decoded metadata, directory references, return/error categories, and full GET bytes. Single-process barriers are insufficient evidence for the network-partition cases.
| Interleaving | Unique permitted outcome |
|---|---|
| A pauses after coordinator verification; A loses lock quorum while data RPC remains reachable; B chooses and publishes; A resumes rename. | A cannot become the chosen successor for B's predecessor or overwrite B on a disk that applied B. Isolated older materialization cannot vote as current. Read B exactly; reconcile laggards under B's decision. |
| A's publication result is unknown; B succeeds; A's rollback/cleanup resumes. | Recover A's decision. Never restore/delete B's metadata or referenced data; retire only A-owned unchosen paths or separately authorized dead directories. |
| Reader acquires generation G, consumes part 1; replacement retires G; GC attempts to delete part 2's directory. | The reader's valid disk protection defers GC; otherwise the reader fails before its next open. Never silently serve another generation or delete a directory still covered by valid protection. Repeat across reader and disk restarts. |
| A and B each stage or partially publish on two of four disks; one coordinator dies; all disks return. | The promise/accept recovery rule preserves any chosen value or adopts the highest accepted candidate and finishes that slot. No guessing from UUID order, no permanent exact-CAS split, no replacement of a chosen value. |
| Process dies after staging sync, intent sync, data rename, metadata replace, directory sync, or accepted/decision reply. | Reopen durable records before readiness. Unchosen work stays private; chosen work is replayed forward; lost ACK resolves to the original operation identity. Torn/insufficient evidence fails closed. |
| Null version becomes a delete marker; a noncurrent version is deleted; delayed heal/metadata write resumes. | Whole-object predecessor no longer matches. Exact repair uses the chosen version set; no resurrection, marker-to-object conversion, or reset to never-present. |
| Old coordinator reaches a new strict disk, new coordinator reaches an old disk, voter restarts, or transport strictness changes. | Compatibility behavior only in an un-enrolled namespace. Strict admission fails until a fresh complete proof and supported startup gate exist; no zero/missing-field fallback. |
| Replacement disk is empty, or a restored minority disk has an old promise and old metadata. | It cannot vote as an initialized authority. Catch up chosen state and reconstruct data under a fresh incarnation; stale requests cannot bypass enrollment by presenting absent metadata. |
| Quota settlement for A arrives after B commits; prepared pool refetch sees B instead of A. | Only A's original chosen outcome may settle A's reservation; B is unaffected. Prepared A cannot supply metadata/data for B without a new validated preparation. |
## Required Implementation Boundaries
These are durable ownership and acceptance boundaries, not permission to close the disk-fencing work before the protocol exists. The decision-model and availability changes require architecture review before production implementation. No unrelated external consensus service or full-manifest rewrite is authorized by this contract.
| Boundary | Required implementation and exit evidence |
|---|---|
| Durable authority substrate | Injected lock-participant persistence; typed ballot/configuration/decision values; prepare/accept/recover state machine; restart-safe proposer identities; corruption and voter replacement handling. Model/exhaustively test two competing proposers, lost replies, minority recovery, and every durable transition. The same-slot different-value property must be impossible. |
| Disk publication boundary | Separate private preparation from publication; implement write-ahead intent, decision receipts, atomic guarded mutation, idempotent recovery, and directory retirement. Include inline/non-inline, every crash point, canceled blocking syscalls, ACK loss, and empty/lagging disks. Existing rollback helpers cannot remain an unguarded alternate route. |
| Writer and read integration | Route every writer in the table and every strict read/scan decision through the authority; preserve data quorum and S3 version semantics. Bind prepared reads, MPU ownership transfer, tier operations, and quota outcomes. Demonstrate no raw metadata entry point bypasses strict mode. |
| Fleet activation | Deploy the startup downgrade barrier first; import legacy/absent heads during quiescence; implement configuration/proof revocation and both RPC encodings. Run real multi-node lock/data-plane partitions and mixed-binary/restart tests. Only then can strict E03 acceptance run and activation be considered. |
The conservative immediate action is to keep the existing compatibility behavior and improve its local convergence/recovery independently. Those fixes must describe their smaller guarantee and must not advertise E03's distributed safety. A strict-only local CAS helper can be built behind the inactive capability boundary, but it cannot enable the feature or close the authority work.
## Performance And Activation Criteria
Measure the existing implementation and the full proposed path on identical machines, disk/filesystem, durability settings, network, object population, concurrency, and warmup. Include single hot-key and many-key 4 KiB PUT, 1 MiB PUT, metadata-only writes, and CompleteMultipartUpload with fixed part counts. Report throughput, p50/p95/p99, peak retained preparation/recovery bytes, recovery time, per-operation RPCs/fsyncs, and the object mutation critical-section duration. Include a slow minority disk, one voter loss, and restart recovery; a throughput result alone is insufficient.
The unoptimized proposal adds a lock-quorum promise round and an accept round with durable writes, plus decision learning/publication and local intent/applied-state persistence. Wdata staging remains separate. Read resolution may add an authority quorum round. Record actual overlapping rounds and fsync group commits; do not claim these costs disappear because the existing lock RPC is reused. Never hold a global lock across shard I/O, wait for all disks on the successful path, or weaken fsync/bitrot/quorum to recover throughput.
Activation requires all failure scenarios to pass with no acknowledged-data loss or wrong-generation read; no unexplained RPC/fsync amplification beyond the implemented phase budget; and an explicit performance acceptance recorded with the review. Use a conservative review trigger of more than 10% throughput loss or 15% p99 growth in any fixed-workload comparison: exceeding it blocks default activation until the architecture/operations owners accept the measured tradeoff or the implementation removes it. These are proposed rollout budgets, not measurements or performance claims. Without a reproducible baseline, leave strict mode unavailable.
docs/operations/tier-ilm-debugging.md
+61
View File
@@ -16,6 +16,7 @@
| Persisted free-version scan and re-enqueue after local-first expiry | `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs` |
| Fenced free-version remote delete, local-marker cleanup, and rescan | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` (`cleanup_free_version_exact`) |
| Durable manual transition job/task/result records | `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs` |
| Dormant tier validation probe intent format and read-only core inspection | `crates/ecstore/src/services/tier/tier_probe_intent.rs` |
| Manual run/status/cancel and transition-transaction reconcile admin routes | `rustfs/src/admin/handlers/ilm_transition.rs` |
| `ObjectInfo` / `TransitionedObject` types | `crates/ecstore/src/object_api/types.rs` |
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
@@ -119,6 +120,12 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --dry-run
rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-objects 1000 --max-duration-seconds 30
```
## Validation probe crash recovery status
Tier Add, Edit, and Verify currently validate a destination with a unique `rustfs-tier-probe-<uuid>` object and perform bounded compensation while the process remains alive. The `rustfs-tier-probe-intent-v1` decoder, canonical durable namespace, conditional storage primitives, state machine, and crate-level inspection type are present only as a dormant foundation. No validation path writes this record, no startup or periodic recovery scans it, and no admin HTTP route exposes it. V1 requires the owner to remain exactly equal to the immutable creator; takeover would require a new schema with explicit proof. Both durable writing and destructive recovery remain disabled until the fleet capability, operation-generation revalidation, provider timeout, retention, and operator contracts are approved.
Do not search the internal metadata bucket for these records as evidence that validation is crash recoverable: a current server does not create them. If a process is killed after the remote probe PUT but before cleanup, inspect the destination provider manually and retain ambiguous candidates. Never delete an empty or guessed version, and do not hand-create a probe intent to authorize cleanup.
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
## Reconcile an unknown transition upload
@@ -154,6 +161,60 @@ Historical transition transactions in `upload_outcome_unknown` state can use an
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
## Inspect and disposition retained recovery records
This section describes an **approved target that is not implemented yet**. Current servers do not expose the routes below and continue to quarantine tier-delete journal v1/v2 records. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees.
The approved read-only inventory is bounded and paginated:
```text
GET /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
GET /rustfs/admin/v3/ilm/recovery/records/<control-id>
```
List and redacted inspect require `admin:ListTier`. The server reconstructs the canonical source identity, strongly reads every authoritative copy, and reports one logical record with its schema, classification (`retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`), stable reason code, copy/content digests, retry deadline/counters, fleet readiness, scan completeness, and decommission coverage. It does not return raw legacy bytes, object/version names, endpoints, credentials, or provider error text in the default JSON. Incomplete pool coverage, divergent copies, a missing ETag, corruption, or a truncated page without a continuation marker is fail-closed and cannot produce an actionable receipt.
Inspect returns a 15-minute opaque observation receipt. It binds the authenticated actor, canonical record, every source copy/ETag/digest, topology/fleet generation, requested action class, issue/expiry time, and nonce. The receipt prevents a stale request from widening its target; it is not cleanup authority.
For a strictly decoded v1/v2 tier-delete journal, the approved evidence-preserving flow is:
1. Inspect the exact record and independently decide whether retaining the local cleanup obligation is still useful.
2. With `admin:SetTier`, create an immutable server-side export from the current observation receipt. The export contains the exact raw journal bytes and copy manifest, is installed create-only at the canonical digest-derived export ID, strongly read back, and downloaded through a no-store attachment response:
```text
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
{ "action": "export", "observation_receipt": "<opaque>" }
GET /rustfs/admin/v3/ilm/recovery/exports/<export-id>
```
3. Only after preserving that export, submit a fresh exact disposition with `admin:SetTier`:
```json
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
{
"action": "abandon_remote_cleanup",
"confirm": true,
"acknowledge_remote_cleanup_abandoned": true,
"observation_receipt": "<opaque>",
"export_id": "<export-id>",
"export_sha256": "<sha256>",
"reason_code": "<bounded-operator-reason>"
}
```
The last action removes only the exact local v1/v2 journal generations by per-copy `If-Match` after a durable `Prepared` disposition receipt and fresh all-member capability proof. The receipt advances `Prepared -> Applying -> Completed` and records a monotonic per-copy `confirmed_absent` set. If the server deletes copy A and crashes before recording progress, recovery may confirm A absent under the unchanged source/control, topology, process-epoch, migration, and decommission proofs, persist that progress, and continue with still-exact copy B. A replacement ETag is always a conflict; recovery never widens the immutable copy manifest.
The action never creates a tier client, probes a backend, or issues remote PUT/GET/DELETE. Its meaning is deliberately narrow: the operator accepts that remote storage may leak and abandons RustFS cleanup after preserving evidence. A changed copy, active decommission, missing member, topology/process restart, incomplete read, or uncertain replacement proof retains the evidence. Success requires every bound copy be durably confirmed absent, a fresh all-member/decommission proof, and the disposition receipt durably `Completed`; response loss resumes only the same canonical operation ID.
Canonical replay of an identical export/disposition consumes no new quota. New operations require a complete artifact inventory and are refused before source mutation when the projected retained total, including the fully encoded candidate, would exceed 10,000 exports, 10,000 disposition receipts, 1 GiB of encoded export data, or 256 MiB of encoded control/disposition data. The quota decision, create-only installation, and exact readback share one cluster-scoped admission WRITE lock. That lock is always acquired before control/source/disposition and physical metadata locks and is released before disposition `Applying` or any source deletion; callers never acquire it while holding those inner guards. A crash before installation consumes no capacity, and lost installation response is resolved by canonical readback under the same serialized order, so concurrent nodes cannot oversubscribe a stale snapshot. Admission is also limited to ten new creations per actor per minute, 100 cluster-wide per minute, 32 concurrent exports, and eight concurrent dispositions. Capacity pressure never evicts recovery evidence or blocks ordinary object I/O; the collector examines at most 100 terminal artifacts per minute.
Malformed/unsupported records and journal v3-v6 cannot use abandon. Known-version and v6 manifest ownership must converge through their normal exact recovery protocol. Operators may inspect, export, and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or version semantics.
Automatic retry state survives restart. Retryable transport/quorum failures use a 60-second exponential base capped at one hour and a deterministic 80-to-100-percent multiplier, so jitter never increases the capped delay. After 32 consecutive failures or seven days from the first persisted failure, automatic work stops at `operator_required`. Unsupported or ambiguous evidence goes directly to `retained_ambiguous`/`operator_required`; age alone never deletes it. Resolved controls, immutable exports, and completed disposition receipts have minimum 30-day, 90-day, and 365-day retention respectively, and are collected only after exact source absence, decommission, successor, and audit checks.
The full schema, lease, mixed-version, retry, privacy, and metric requirements are in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#bounded-recovery-control-and-operator-disposition).
## Reconcile legacy transition-version metadata
This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version.
docs/testing/README.md
+2
View File
@@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md).
The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
## Naming conventions
### Reserved test-name substrings (migration gate)
docs/testing/ci-gates.md
+10
View File
@@ -109,3 +109,13 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches
## Change checklist
Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here.
## ECStore invariant selection
The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact.
The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered.
The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability.
Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review.
docs/testing/ecstore-validation-suite-design.md
+16
View File
@@ -54,6 +54,22 @@ Fail-closed invariants every row enforces:
Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection.
### PUT completion fixtures
`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking.
The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup.
| Fixture | Completion boundary |
|---|---|
| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. |
| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. |
| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. |
| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. |
| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. |
Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks.
### Coverage gate
`full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is:
docs/testing/scanner-checkpoint-fixture.md
+22
View File
@@ -0,0 +1,22 @@
# Scanner Checkpoint Fixture
The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
Run the fixture and confirm the test filter selects a nonzero number of tests:
```sh
cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list
RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture
```
The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well:
```sh
RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture
```
A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome.
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches.
rustfs/src/admin/handlers/replication.rs
+11 -1
View File
@@ -121,6 +121,11 @@ fn map_bucket_target_error(err: BucketTargetError) -> S3Error {
| BucketTargetError::BucketRemoteRemoveDisallowed { .. } => {
S3Error::with_message(S3ErrorCode::InvalidRequest, err.to_string())
}
// A stored target configuration this node cannot decode is a
// server-side data fault, not a bad request (rustfs/backlog#2282).
BucketTargetError::BucketRemoteTargetsUnreadable { .. } => {
S3Error::with_message(S3ErrorCode::InternalError, err.to_string())
}
BucketTargetError::Io(io_err) => S3Error::with_message(S3ErrorCode::InternalError, io_err.to_string()),
}
}
@@ -753,7 +758,12 @@ impl Operation for ListRemoteTargetHandler {
.map_err(ApiError::from)?;
let sys = BucketTargetSys::get();
let targets = sys.list_targets(bucket, "").await;
// An unreadable targets configuration must not be reported as an
// empty target list (rustfs/backlog#2282).
let targets = sys.list_targets(bucket, "").await.map_err(|e| {
error!("list remote targets failed: {}", e);
map_bucket_target_error(e)
})?;
let targets: Vec<_> = targets
.iter()
rustfs/src/app/bucket_list_through.rs
+354 -5
View File
@@ -252,8 +252,16 @@ pub(crate) async fn merged_list_objects_v2(
.filter(|entry| merger.accepts(&entry.key().name))
.collect();
let keys: Vec<ListEntryKey> = kept.iter().map(SideEntry::key).collect();
if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) {
match fetch.side {
MergeSide::Source => {
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
continue;
}
MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
}
}
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
merger.push_page(fetch.side, keys, is_truncated, next_token);
}
let outcome = merger.finish();
@@ -340,10 +348,12 @@ async fn fetch_source_page(
continuation_token: token,
max_keys: params.max_keys,
},
// Everything under `filter.prefix` rolls into one common prefix, so a
// single bounded listing settles whether it exists.
// Everything under `filter.prefix` rolls into one common prefix. An
// empty truncated probe must still follow its cursor before declaring
// that prefix absent.
SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest {
prefix: Some(probe_prefix.as_str()),
continuation_token: token,
max_keys: 1,
..Default::default()
},
@@ -368,8 +378,8 @@ async fn fetch_source_page(
} else {
Vec::new()
},
false,
None,
!exists && page.is_truncated,
if exists { None } else { page.next_continuation_token },
))
}
_ => {
@@ -417,6 +427,17 @@ async fn local_delete_markers(store: &Arc<ECStore>, bucket: &str, keys: &[String
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
};
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
use crate::app::storage_api::test::StoragePutObjReader;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::contract::object::ObjectIO as _;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
fn token(local: Option<&str>, local_done: bool) -> ListThroughToken {
ListThroughToken {
@@ -526,4 +547,332 @@ mod tests {
assert!(degraded);
assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local));
}
/// Serves exactly the scripted S3 pages and joins every connection before
/// returning. A source retry or unexpected operation fails the test.
async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind listing source");
let address = listener.local_addr().expect("listing source address");
let server = tokio::spawn(async move {
let mut requests = Vec::new();
for body in pages {
let (mut stream, _) = listener.accept().await.expect("accept source listing");
let mut request = Vec::new();
let mut chunk = [0; 4096];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let count = stream.read(&mut chunk).await.expect("read signed listing request");
assert!(count > 0, "source request must include complete headers");
request.extend_from_slice(&chunk[..count]);
assert!(request.len() <= 32 * 1024, "listing request headers must be bounded");
}
let first_line = String::from_utf8_lossy(&request)
.lines()
.next()
.expect("request line")
.to_string();
// The SDK joins the bucket endpoint with the LIST operation's `/` path.
assert!(
first_line.starts_with("GET /source-bucket/?"),
"expected a path-style bucket-root LIST request, got {first_line:?}"
);
assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}");
requests.push(first_line);
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes()).await.expect("write source page");
stream.shutdown().await.expect("finish source response");
}
requests
});
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
}
fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
let next = next
.map(|token| format!("<NextContinuationToken>{token}</NextContinuationToken>"))
.unwrap_or_default();
let contents = key
.map(|key| format!("<Contents><Key>{key}</Key><Size>1</Size></Contents>"))
.unwrap_or_default();
format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next}{contents}</ListBucketResult>"
)
}
struct ListThroughTestState {
bucket: String,
module_enabled: bool,
}
impl Drop for ListThroughTestState {
fn drop(&mut self) {
let sys = OnDemandMigrationSys::get();
sys.remove(&self.bucket);
sys.set_module_enabled(self.module_enabled);
}
}
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let store = shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create list-through bucket");
store
.put_object(
&bucket,
"z-local",
&mut StoragePutObjReader::from_vec(vec![1]),
&StorageObjectOptions::default(),
)
.await
.expect("seed real local listing");
let (endpoint, server) = scripted_list_source(pages).await;
let sys = OnDemandMigrationSys::get();
let _state_guard = ListThroughTestState {
bucket: bucket.clone(),
module_enabled: sys.is_module_enabled(),
};
sys.set_module_enabled(true);
let config = OnDemandMigrationConfig {
version: 1,
enabled: true,
source: SourceConfig {
provider: Provider::Minio,
endpoint: Some(endpoint),
region: "us-east-1".into(),
bucket: "source-bucket".into(),
path_style: PathStyle::Path,
credentials: Some(SourceCredentials {
access_key: "test-access".into(),
secret_key: "test-secret".into(),
session_token: None,
}),
tls: TlsConfig::default(),
},
filter: FilterConfig {
prefix: filter_prefix.map(str::to_string),
..Default::default()
},
policy: PolicyConfig {
list_through: true,
source_error: policy,
..Default::default()
},
};
sys.apply(&bucket, Some(&config)).await;
assert!(
sys.state(&bucket).expect("ODM state installed").client().is_ok(),
"fake source client must build"
);
let continuation_token = resume_source.map(|source| {
let token = ListThroughToken {
t: "odm-list".into(),
v: 1,
local: None,
local_done: false,
source: Some(source.into()),
source_done: false,
last_key: None,
};
base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
});
let input = ListObjectsV2Input {
bucket,
max_keys: Some(2),
continuation_token,
delimiter: filter_prefix.map(|_| "/".to_string()),
encoding_type: None,
expected_bucket_owner: None,
fetch_owner: None,
optional_object_attributes: None,
prefix: None,
request_payer: None,
start_after: None,
};
let request = S3Request {
input,
method: http::Method::GET,
uri: http::Uri::from_static("/?list-type=2"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let result = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
)
.await
.expect("listing must complete within its bounded source budget");
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("source connections must finish")
.expect("source server must not panic");
(result, requests)
}
#[test]
#[serial_test::serial]
fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
run_large_stack_test("list-through-source-policy", || async {
temp_env::async_with_vars(
[
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
for next in [None, Some(""), Some("stuck")] {
for key in [None, Some("a-source")] {
let (result, requests) =
source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await;
assert_eq!(requests.len(), 1, "a malformed source page must not be retried");
assert!(requests[0].contains("continuation-token=stuck"));
assert_source_policy_result(result, policy);
}
}
let (result, requests) = source_policy_request(
vec![
source_xml(Some("stuck"), true, Some("a-source")),
source_xml(Some("stuck"), true, None),
],
policy,
None,
None,
)
.await;
assert_eq!(requests.len(), 2, "the failure must occur during a real refill");
assert!(!requests[0].contains("continuation-token="));
assert!(requests[1].contains("continuation-token=stuck"));
assert_source_policy_result(result, policy);
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() {
run_large_stack_test("list-through-empty-source-pages", || async {
temp_env::async_with_vars(
[
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for filter_prefix in [None, Some("photos/2024/")] {
let source_key = if filter_prefix.is_some() {
"photos/2024/a-source"
} else {
"a-source"
};
let (result, requests) = source_policy_request(
vec![
source_xml(Some("opaque-next"), true, None),
source_xml(None, false, Some(source_key)),
],
SourceErrorPolicy::Propagate,
None,
filter_prefix,
)
.await;
assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor");
assert!(requests[1].contains("continuation-token=opaque-next"));
let response = result.expect("empty progressing source page is valid");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = response.output;
let objects: Vec<_> = output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("listed object key"))
.collect();
if filter_prefix.is_some() {
assert_eq!(objects, vec!["z-local"]);
assert_eq!(
output
.common_prefixes
.unwrap_or_default()
.into_iter()
.map(|prefix| prefix.prefix.expect("rolled-up prefix"))
.collect::<Vec<_>>(),
vec!["photos/"]
);
} else {
assert_eq!(objects, vec!["a-source", "z-local"]);
assert!(output.common_prefixes.unwrap_or_default().is_empty());
}
assert_eq!(output.key_count, Some(2));
assert_eq!(output.is_truncated, Some(false));
assert!(output.next_continuation_token.is_none());
}
},
)
.await;
});
}
fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
match policy {
SourceErrorPolicy::Propagate => {
let error = result.expect_err("propagate must expose malformed pagination");
assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY));
assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into()));
assert_eq!(error.message(), Some("invalid_pagination"));
}
SourceErrorPolicy::NotFound => {
let response = result.expect("not_found must preserve the local listing");
assert_eq!(
response
.headers
.get("x-rustfs-on-demand-migration-list")
.expect("local_only header"),
"local_only"
);
let output = response.output;
assert_eq!(
output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("local key"))
.collect::<Vec<_>>(),
vec!["z-local"]
);
assert_eq!(output.is_truncated, Some(false));
assert_eq!(output.key_count, Some(1));
assert!(output.next_continuation_token.is_none());
}
}
}
}
rustfs/src/app/object/copy.rs
+45 -1
View File
@@ -394,7 +394,7 @@ impl DefaultObjectUsecase {
// Bucket metadata uses the bucket name as its namespace-lock key. Load
// every copy-time bucket snapshot before a same-object key can collide
// with that key (for example, copying `bucket/bucket` onto itself).
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?;
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
if cp_src_dst_same && key == bucket {
dst_opts.object_lock_config_snapshot =
@@ -1388,4 +1388,48 @@ mod tests {
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
#[tokio::test]
#[serial_test::serial]
async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let (store, context) = real_store_test_context().await;
let bucket = format!("copy-sse-unreadable-{}", Uuid::new_v4());
let source = "source.bin";
let destination = "destination.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("unreadable-encryption copy bucket must be created");
let mut reader = PutObjReader::from_vec(b"copied while the bucket still had a readable configuration".to_vec());
store
.put_object(&bucket, source, &mut reader, &ObjectOptions::default())
.await
.expect("copy source object must be written");
install_unreadable_bucket_sse_config(&bucket).await;
let input = CopyObjectInput::builder()
.copy_source(CopySource::Bucket {
bucket: bucket.clone().into(),
key: source.into(),
version_id: None,
})
.bucket(bucket.clone())
.key(destination.to_string())
.build()
.expect("copy input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the copy");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
let lookup_err = store
.get_object_info(&bucket, destination, &ObjectOptions::default())
.await
.expect_err("a refused copy must not leave a destination object behind");
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
}
}
rustfs/src/app/object/extract.rs
+1 -1
View File
@@ -2037,7 +2037,7 @@ impl DefaultObjectUsecase {
let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5);
let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?);
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?;
let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse(
bucket_sse_config.as_ref().map(|(config, _timestamp)| config),
original_sse,
rustfs/src/app/object/put.rs
+119 -1
View File
@@ -1485,8 +1485,9 @@ impl DefaultObjectUsecase {
};
let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await;
rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start);
let bucket_sse_config = bucket_sse_config?;
debug!(
target: "rustfs::app::object_usecase",
component = "app",
@@ -3912,4 +3913,121 @@ mod tests {
.expect_err("writes after the zero-byte quota update must be denied");
assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 }));
}
#[tokio::test]
#[serial_test::serial]
async fn execute_put_object_refuses_a_bucket_whose_encryption_config_is_unreadable() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let (store, context) = real_store_test_context().await;
let bucket = format!("put-sse-unreadable-{}", Uuid::new_v4());
let object = "object.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("unreadable-encryption PUT bucket must be created");
install_unreadable_bucket_sse_config(&bucket).await;
let payload = Bytes::from_static(b"an operator mandated encryption for this bucket");
let input = PutObjectInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the write");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
let lookup_err = store
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect_err("a refused PUT must not leave an object behind");
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
}
#[tokio::test]
#[serial_test::serial]
async fn execute_put_object_still_writes_plaintext_without_bucket_encryption() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let (store, context) = real_store_test_context().await;
let bucket = format!("put-sse-absent-{}", Uuid::new_v4());
let object = "object.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("plaintext PUT bucket must be created");
let payload = Bytes::from_static(b"no default encryption is configured for this bucket");
let input = PutObjectInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
.build()
.expect("PUT input must build");
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
.await
.expect("a bucket without default encryption must still accept a plaintext write");
let stored = store
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("the plaintext object must be readable");
assert_eq!(stored.size, i64::try_from(payload.len()).expect("test payload length must fit i64"));
assert!(
!stored
.user_defined
.keys()
.any(|key| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION)
|| key.starts_with("x-rustfs-encryption-")
|| key.starts_with("x-minio-encryption-")),
"the object must carry no encryption metadata: {:?}",
stored.user_defined
);
}
#[tokio::test]
#[serial_test::serial]
async fn execute_put_object_extract_refuses_a_bucket_whose_encryption_config_is_unreadable() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let (store, context) = real_store_test_context().await;
let bucket = format!("extract-sse-unreadable-{}", Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("unreadable-encryption extract bucket must be created");
install_unreadable_bucket_sse_config(&bucket).await;
let payload = Bytes::from_static(b"archive bytes that must never be unpacked in plaintext");
let input = PutObjectInput::builder()
.bucket(bucket.clone())
.key("archive.tar".to_string())
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
.build()
.expect("extract PUT input must build");
let mut req = build_request(input, Method::PUT);
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
.await
.expect_err("an unreadable bucket encryption configuration must refuse the extract upload");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
let lookup_err = store
.get_object_info(&bucket, "archive.tar", &ObjectOptions::default())
.await
.expect_err("a refused extract upload must not leave an object behind");
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
}
}
rustfs/src/app/object/shared.rs
+123
View File
@@ -269,6 +269,129 @@ pub(super) fn resolve_bucket_default_sse(
(effective_sse, effective_kms_key_id)
}
/// The bucket's default encryption configuration for a write path.
///
/// `Ok(None)` carries one meaning only — this bucket has no default encryption
/// — and the write proceeds in plaintext exactly as before. Every other
/// outcome refuses the write rather than collapsing onto that same value: an
/// encryption blob that exists but cannot be read fails closed in
/// `get_sse_config` since rustfs/rustfs#7172, and swallowing that error here
/// stores plaintext into a bucket whose operator mandated encryption, with
/// nothing returned to the client and nothing in the object to tell it apart
/// afterwards (rustfs/backlog#2287).
///
/// The states the lookup can report, and what each one does:
///
/// * configured and readable — apply the bucket default;
/// * no encryption blob at all, including a bucket that does not exist and a
/// bucket whose metadata document is absent — `ConfigNotFound`, so a cold
/// cache and a missing bucket are never turned into a refusal, and the write
/// still fails later with its own `NoSuchBucket`;
/// * blob present but unparseable — deterministic, so retrying cannot help;
/// surfaces as `InternalError` until an operator repairs or removes it;
/// * the metadata read itself failed (namespace lock, quorum, disk, an
/// uninitialized metadata system) — transient, and the typed error maps to
/// the retryable `ServiceUnavailable`.
///
/// The last two are distinguished by the typed error the accessor returns, not
/// re-derived here: [`ApiError`] already separates them. This mirrors
/// `prepare_sse_configuration` in `storage::sse`, the resolver the multipart
/// writer uses, which has always failed closed on the same lookup.
pub(super) async fn load_bucket_default_sse_config(
bucket: &str,
) -> S3Result<Option<(ServerSideEncryptionConfiguration, OffsetDateTime)>> {
classify_bucket_default_sse_lookup(bucket, metadata_sys::get_sse_config(bucket).await)
}
fn classify_bucket_default_sse_lookup(
bucket: &str,
lookup: Result<(ServerSideEncryptionConfiguration, OffsetDateTime), StorageError>,
) -> S3Result<Option<(ServerSideEncryptionConfiguration, OffsetDateTime)>> {
match lookup {
Ok(config) => Ok(Some(config)),
Err(err) if err == StorageError::ConfigNotFound => Ok(None),
Err(err) => {
let api_error = ApiError::from(err);
error!(
event = "bucket_sse_config_lookup_failed",
component = LOG_COMPONENT_APP,
subsystem = LOG_SUBSYSTEM_OBJECT,
result = "write_refused",
bucket = %bucket,
code = %api_error.code.as_str(),
error = %api_error,
"Bucket default encryption is unreadable; refusing the write instead of storing plaintext"
);
Err(api_error.into())
}
}
}
#[cfg(test)]
mod bucket_default_sse_lookup_tests {
use super::*;
use s3s::dto::{ServerSideEncryptionByDefault, ServerSideEncryptionRule};
use time::OffsetDateTime;
fn sse_config() -> ServerSideEncryptionConfiguration {
ServerSideEncryptionConfiguration {
rules: vec![ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault {
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AES256),
kms_master_key_id: None,
}),
blocked_encryption_types: None,
bucket_key_enabled: None,
}],
}
}
#[test]
fn an_absent_configuration_still_writes_plaintext() {
let resolved = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ConfigNotFound))
.expect("a bucket without default encryption must keep writing plaintext");
assert!(resolved.is_none());
assert_eq!(resolve_bucket_default_sse(None, None, None, false), (None, None));
}
#[test]
fn a_readable_configuration_is_returned() {
let resolved = classify_bucket_default_sse_lookup("bucket", Ok((sse_config(), OffsetDateTime::UNIX_EPOCH)))
.expect("a readable configuration must not refuse the write")
.expect("a readable configuration must be applied");
assert_eq!(resolved.0.rules.len(), 1);
}
#[test]
fn an_unreadable_configuration_refuses_the_write() {
let err = classify_bucket_default_sse_lookup(
"bucket",
Err(StorageError::other("persisted bucket encryption configuration is invalid")),
)
.expect_err("a corrupt encryption blob must never degrade to plaintext");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn an_unavailable_metadata_read_refuses_the_write_as_retryable() {
let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum))
.expect_err("an unreadable metadata subsystem must never degrade to plaintext");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
}
#[test]
fn a_missing_bucket_keeps_its_own_error() {
let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::BucketNotFound("bucket".to_string())))
.expect_err("a bucket-not-found lookup must not be reported as an encryption failure");
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
}
}
#[cfg(test)]
mod deadlock_request_guard_tests {
use super::DeadlockRequestGuard;
rustfs/src/app/object/test_support.rs
+33
View File
@@ -96,3 +96,36 @@ pub(super) fn real_cold_fill_plan(
};
plan
}
/// A store with an ambient `AppContext`, for tests that drive a handler end to
/// end without the object-data-cache overrides of
/// [`real_cold_fill_test_context`].
pub(super) async fn real_store_test_context() -> (Arc<ECStore>, Arc<AppContext>) {
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
if current_app_context().is_none() {
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
}
let ambient = current_app_context().expect("real-store tests require an ambient AppContext");
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
(store, context)
}
/// Leave the bucket in the state a damaged encryption blob produces: the raw
/// document is retained and the typed configuration stays `None`, which is the
/// durable "exists but cannot be read" signal `get_sse_config` fails closed on
/// (rustfs/rustfs#7172).
pub(super) async fn install_unreadable_bucket_sse_config(bucket: &str) {
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system must be initialized");
let metadata = {
let sys = sys.read().await;
sys.get(bucket).await.expect("bucket metadata must be cached")
};
let mut metadata = (*metadata).clone();
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration>truncated".to_vec();
metadata.sse_config = None;
set_bucket_metadata(bucket.to_string(), metadata)
.await
.expect("unreadable bucket encryption configuration must be installed");
}
rustfs/src/app/storage_api.rs
+5 -3
View File
@@ -29,11 +29,13 @@ pub(crate) fn EndpointServerPools(
pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
};
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
#[cfg(test)]
pub(crate) use s3s::{S3Request, S3Response};
}
pub(crate) mod admin {
rustfs/src/version.rs
+6 -162
View File
@@ -14,8 +14,6 @@
use const_str::concat;
use shadow_rs::shadow;
use std::path::Path;
use std::process::Command;
shadow!(build);
@@ -47,10 +45,6 @@ pub const DISPLAY_VERSION: &str = {
type VersionParseResult = Result<(u32, u32, u32, Option<String>), Box<dyn std::error::Error>>;
fn build_version_override() -> Option<&'static str> {
BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty())
}
fn version_ref(version: &str) -> String {
if version.starts_with("refs/tags/") || version.starts_with('@') {
version.to_string()
@@ -61,91 +55,7 @@ fn version_ref(version: &str) -> String {
#[allow(clippy::const_is_empty)]
pub fn get_version() -> String {
if let Some(version) = build_version_override() {
return version_ref(version);
}
// Get the latest tag
if let Ok(latest_tag) = get_latest_tag() {
// Check if current commit is newer than the latest tag
if is_head_newer_than_tag(&latest_tag) {
// If current commit is newer, increment the version number
if let Ok(new_version) = increment_version(&latest_tag) {
return format!("refs/tags/{new_version}");
}
}
// If current commit is the latest tag, or version increment failed, return current tag
return format!("refs/tags/{latest_tag}");
}
// If no tag exists, use original logic
if !build::TAG.is_empty() {
format!("refs/tags/{}", build::TAG)
} else if !build::SHORT_COMMIT.is_empty() {
format!("@{}", build::SHORT_COMMIT)
} else {
format!("refs/tags/{}", build::PKG_VERSION)
}
}
/// Get the latest git tag
fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?;
if output.status.success() {
let tag = String::from_utf8(output.stdout)?;
Ok(tag.trim().to_string())
} else {
Err("Failed to get latest tag".into())
}
}
/// Check if current HEAD is newer than specified tag
fn is_head_newer_than_tag(tag: &str) -> bool {
is_head_newer_than_tag_in(Path::new("."), tag)
}
fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool {
let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output();
let tag_commit = Command::new("git")
.current_dir(repo)
.args(["rev-list", "-n", "1", tag])
.output();
let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else {
return false;
};
if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout {
return false;
}
let output = Command::new("git")
.current_dir(repo)
.args(["merge-base", "--is-ancestor", tag, "HEAD"])
.output();
match output {
Ok(result) => result.status.success(),
Err(_) => false,
}
}
/// Increment version number (increase patch version)
fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>> {
// Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
let (major, minor, patch, pre_release) = parse_version(version)?;
// If there's a pre-release identifier, increment the pre-release version number
if let Some(pre) = pre_release
&& let Some(new_pre) = increment_pre_release(&pre)
{
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
}
// Otherwise increment patch version number
Ok(format!("{major}.{minor}.{}", patch + 1))
version_ref(DISPLAY_VERSION)
}
/// Parse version number
@@ -166,28 +76,6 @@ pub fn parse_version(version: &str) -> VersionParseResult {
Ok((major, minor, patch, pre_release))
}
/// Increment pre-release version number
fn increment_pre_release(pre_release: &str) -> Option<String> {
// Handle pre-release versions like "alpha.19"
let parts: Vec<&str> = pre_release.split('.').collect();
if parts.len() == 2
&& let Ok(num) = parts[1].parse::<u32>()
{
return Some(format!("{}.{}", parts[0], num + 1));
}
// Handle pre-release versions like "alpha19"
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
let prefix = &pre_release[..=pos];
let suffix = &pre_release[pos + 1..];
if let Ok(num) = suffix.parse::<u32>() {
return Some(format!("{prefix}{}", num + 1));
}
}
None
}
/// Clean version string - removes common prefixes
pub fn clean_version(version: &str) -> String {
version
@@ -284,34 +172,6 @@ mod tests {
use super::*;
use tracing::debug;
fn run_git(repo: &Path, args: &[&str]) {
let status = Command::new("git").current_dir(repo).args(args).status().unwrap();
assert!(status.success(), "git command failed: git {}", args.join(" "));
}
#[test]
fn test_is_head_newer_than_tag_requires_strict_descendant() {
let repo = tempfile::tempdir().unwrap();
run_git(repo.path(), &["init", "--quiet"]);
run_git(repo.path(), &["config", "user.name", "RustFS Tests"]);
run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]);
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]);
run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]);
assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3"));
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]);
assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3"));
}
#[test]
fn build_version_override_is_used_for_current_version_when_set() {
if let Some(version) = build_version_override() {
assert_eq!(get_version(), version_ref(version));
}
}
#[test]
fn version_ref_keeps_existing_ref_prefixes() {
assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3");
@@ -319,6 +179,11 @@ mod tests {
assert_eq!(version_ref("@abc123"), "@abc123");
}
#[test]
fn get_version_uses_build_metadata() {
assert_eq!(get_version(), version_ref(DISPLAY_VERSION));
}
#[test]
fn test_parse_version() {
// Test standard version parsing
@@ -336,27 +201,6 @@ mod tests {
assert_eq!(pre_release, Some("alpha.19".to_string()));
}
#[test]
fn test_increment_pre_release() {
// Test alpha.19 -> alpha.20
assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string()));
// Test beta.5 -> beta.6
assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string()));
// Test unparsable case
assert_eq!(increment_pre_release("unknown"), None);
}
#[test]
fn test_increment_version() {
// Test pre-release version increment
assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20");
// Test standard version increment
assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1");
}
#[test]
fn test_version_format() {
// Test if version format starts with refs/tags/
scripts/check_test_wiring.py
+78 -1
View File
@@ -764,8 +764,53 @@ def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
return []
def core_requirements(root: Path) -> dict:
data = json.loads((root / ".config/ecstore-required-tests.json").read_text())
if not data["tests"] or not data["fixtures"]:
raise ValueError("core test and fixture requirements must not be empty")
identities = [(test["suite"], test["name"]) for test in data["tests"]]
if len(set(identities)) != len(identities):
raise ValueError("duplicate core test requirement")
return data
def check_core_fixtures(root: Path) -> list[str]:
try:
fixtures = core_requirements(root)["fixtures"]
errors = []
for fixture in fixtures:
path = (root / fixture["path"]).resolve()
if not path.is_relative_to(root.resolve()):
raise ValueError("core fixture path escapes repository")
if not path.is_file():
errors.append(f"{fixture['path']}: required core fixture missing")
elif hashlib.sha256(path.read_bytes()).hexdigest() != fixture["sha256"]:
errors.append(f"{fixture['path']}: core fixture sha256 mismatch")
return errors
except (OSError, KeyError, TypeError, ValueError) as error:
return [f"cannot validate core fixtures: {error}"]
def check_core_listing(root: Path, listing: Path) -> list[str]:
"""Check the existing CI run's selection, not a second filtered test run."""
try:
required = core_requirements(root)["tests"]
suites = json.loads(listing.read_text())["rust-suites"]
if not isinstance(suites, dict):
raise ValueError("rust-suites must be an object")
errors = check_core_fixtures(root)
for test in required:
testcase = suites.get(test["suite"], {}).get("testcases", {}).get(test["name"], {})
if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches":
errors.append(f"{test['invariant']}: required test not selected: {test['suite']}::{test['name']}")
return errors
except (OSError, KeyError, TypeError, ValueError) as error:
return [f"cannot read core nextest listing: {error}"]
def validate(root: Path) -> list[str]:
errors: list[str] = []
errors.extend(check_core_fixtures(root))
errors.extend(check_e2e_modules(root))
errors.extend(check_vault_test_groups(root))
errors.extend(check_ilm_build_budget(root))
@@ -779,6 +824,32 @@ def validate(root: Path) -> list[str]:
class SelfTests(unittest.TestCase):
def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / ".config").mkdir()
fixture = root / "fixture.hex"
fixture.write_text("4142")
requirements = {
"tests": [{"invariant": "commit", "suite": "store", "name": "commit_test"}],
"fixtures": [{"path": "fixture.hex", "sha256": hashlib.sha256(fixture.read_bytes()).hexdigest()}],
}
(root / ".config/ecstore-required-tests.json").write_text(json.dumps(requirements))
listing = root / "listing.json"
good = {"ignored": False, "filter-match": {"status": "matches"}}
for case, testcase in (("selected", good), ("missing", {}), ("ignored", dict(good, ignored=True)),
("filtered", dict(good, **{"filter-match": {"status": "mismatch"}}))):
with self.subTest(case=case):
listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": testcase}}}}))
self.assertEqual(bool(check_core_listing(root, listing)), case != "selected")
listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": good}}}}))
fixture.write_text("4143")
self.assertIn("sha256 mismatch", check_core_listing(root, listing)[0])
fixture.unlink()
self.assertIn("fixture missing", check_core_listing(root, listing)[0])
listing.write_text("not json")
self.assertIn("cannot read", check_core_listing(root, listing)[0])
def test_ilm_lane_keeps_the_measured_cargo_build_budget(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -981,6 +1052,7 @@ class SelfTests(unittest.TestCase):
mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
mock.patch(__name__ + ".check_vault_test_groups", return_value=[]),
mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
mock.patch(__name__ + ".check_core_fixtures", return_value=[]),
mock.patch(__name__ + ".check_runner_selection", return_value=[]),
mock.patch(__name__ + ".check_workflow_readiness", return_value=[]),
mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
@@ -1393,6 +1465,11 @@ def main() -> int:
if sys.argv[1:] == ["--self-test"]:
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
if len(sys.argv) == 3 and sys.argv[1] == "--check-core":
errors = check_core_listing(ROOT, Path(sys.argv[2]))
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1 if errors else 0
if len(sys.argv) == 4 and sys.argv[1] == "--check-profile":
errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3]))
if errors:
@@ -1410,7 +1487,7 @@ def main() -> int:
return 0
if sys.argv[1:]:
print(
"usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING | "
"usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | "
"--update-profile PROFILE LISTING PLATFORM]",
file=sys.stderr,
)
scripts/ecstore-module-lint-register.txt
-20
View File
@@ -20,8 +20,6 @@
crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|clippy::all
crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_must_use
crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables
crates/s3-client/src/api_error_response.rs|clippy::all
crates/s3-client/src/api_error_response.rs|unused_must_use
@@ -74,36 +72,18 @@ crates/ecstore/src/services/event_notification.rs|unused_variables
crates/ecstore/src/services/tier/tier.rs|clippy::all
crates/ecstore/src/services/tier/tier.rs|unused_must_use
crates/ecstore/src/services/tier/tier.rs|unused_variables
crates/ecstore/src/services/tier/tier_admin.rs|clippy::all
crates/ecstore/src/services/tier/tier_admin.rs|unused_must_use
crates/ecstore/src/services/tier/tier_admin.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_aliyun.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_azure.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_gcs.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_minio.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_r2.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_rustfs.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_s3.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables
crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables