Compare commits

..

2 Commits

Author SHA1 Message Date
xiaomage 0de98f8139 fix(ci): use English headers in the upgrade matrix table 2026-09-04 19:21:11 +08:00
xiaomage 28f677d723 ci(upgrade): render an upgrade matrix in the report; fix from default
The upgrade report only ever showed the requested deb URLs and a case
table. The nightly chain runs died installing the OLD package (default
from_version 1.0.0-rc.4-preview.1 has no .deb asset on its release, and
release 1.0.0-rc.4 ships none either), leaving an empty Total: 0 report
with no indication of what was upgraded.

- Default from_version is now 1.0.0-rc.3 (ships rustfs_1.0.0.rc.3_amd64.deb).
  Matches the auto-testing default from PR #32.
- The report generator also parses the [UPG-TOPO] lines the suite now
  emits and renders an 'Upgrade Matrix' section: per topology and KMS
  backend, the versions actually in place before/after (captured via
  'rustfs --version' on the node) and the aggregated result. When the
  suite dies before any topology completes, the matrix says so instead
  of silently showing nothing.
2026-09-04 19:11:39 +08:00
36 changed files with 1160 additions and 7457 deletions
+2 -2
View File
@@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
setup = 'ecstore-large-stack'
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
@@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
setup = 'ecstore-large-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
-56
View File
@@ -152,60 +152,6 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
STEPS_TABLE="/tmp/rustfs-heal-steps.md"
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
steps = {}
order = []
version = None
version_node = None
verdict = None
verdict_detail = ''
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = step_re.match(line)
if m:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
steps[n] = (desc, status) # later lines win (fail after pass)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
if m:
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Step Results\n\n')
if version:
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
out.write(f'- Version under test: **{version}**{node_note}\n')
if verdict:
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
out.write('\n')
out.write('| Step | Description | Result |\n')
out.write('| --- | --- | --- |\n')
for n in sorted(order, key=int):
desc, status = steps[n]
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
PY
{
echo "# RustFS heal test report"
echo ""
@@ -214,8 +160,6 @@ jobs:
echo "- Package: ${PACKAGE_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${STEPS_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
@@ -380,60 +380,6 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md"
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
steps = {}
order = []
version = None
version_node = None
verdict = None
verdict_detail = ''
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = step_re.match(line)
if m:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
steps[n] = (desc, status) # later lines win (fail after pass)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
if m:
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Step Results\n\n')
if version:
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
out.write(f'- Version under test: **{version}**{node_note}\n')
if verdict:
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
out.write('\n')
out.write('| Step | Description | Result |\n')
out.write('| --- | --- | --- |\n')
for n in sorted(order, key=int):
desc, status = steps[n]
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
PY
{
echo "# RustFS pool expansion test report"
echo ""
@@ -443,8 +389,6 @@ jobs:
echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
echo ""
cat "${STEPS_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
Generated
+74 -45
View File
@@ -1328,9 +1328,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.16.0"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223"
checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1368,9 +1368,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.6.3"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
dependencies = [
"base64-simd",
"bytes",
@@ -5103,9 +5103,9 @@ dependencies = [
[[package]]
name = "hickory-net"
version = "0.26.2"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1"
checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183"
dependencies = [
"async-trait",
"cfg-if",
@@ -5127,9 +5127,9 @@ dependencies = [
[[package]]
name = "hickory-proto"
version = "0.26.2"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d"
checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643"
dependencies = [
"data-encoding",
"idna",
@@ -5147,9 +5147,9 @@ dependencies = [
[[package]]
name = "hickory-resolver"
version = "0.26.2"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f"
checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c"
dependencies = [
"cfg-if",
"futures-util",
@@ -5226,9 +5226,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.25.0"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592"
checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
dependencies = [
"arc-swap",
"async-channel",
@@ -5240,6 +5240,7 @@ dependencies = [
"futures-util",
"hdrhistogram",
"hotpath-macros",
"hotpath-meta",
"http 1.5.0",
"libc",
"object 0.36.7",
@@ -5259,15 +5260,30 @@ dependencies = [
[[package]]
name = "hotpath-macros"
version = "0.25.0"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "hotpath-macros-meta"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
[[package]]
name = "hotpath-meta"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
dependencies = [
"hotpath-macros-meta",
]
[[package]]
name = "htmlescape"
version = "0.3.1"
@@ -5626,6 +5642,18 @@ dependencies = [
"tempfile",
]
[[package]]
name = "internal-russh-num-bigint"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8"
dependencies = [
"num-integer",
"num-traits",
"rand 0.10.2",
"rand_core 0.10.1",
]
[[package]]
name = "io-uring"
version = "0.7.14"
@@ -5876,20 +5904,17 @@ dependencies = [
[[package]]
name = "kafka-protocol"
version = "0.18.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda"
checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2"
dependencies = [
"anyhow",
"bytes",
"crc",
"crc32c",
"flate2",
"indexmap 2.14.1",
"lz4",
"snap",
"paste",
"uuid",
"zstd",
]
[[package]]
@@ -6898,8 +6923,6 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
dependencies = [
"num-integer",
"num-traits",
"rand 0.10.2",
"rand_core 0.10.1",
]
[[package]]
@@ -7492,9 +7515,9 @@ dependencies = [
[[package]]
name = "pageant"
version = "0.2.3"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce"
checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
dependencies = [
"base16ct 1.0.0",
"byteorder",
@@ -7583,6 +7606,12 @@ dependencies = [
"phc",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "4.0.1"
@@ -9221,9 +9250,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.63.2"
version = "0.63.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd"
checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031"
dependencies = [
"aes 0.9.3",
"aws-lc-rs",
@@ -9252,12 +9281,13 @@ dependencies = [
"hex-literal",
"hmac 0.13.0",
"inout 0.2.2",
"internal-russh-num-bigint",
"keccak",
"log",
"md5",
"ml-kem",
"module-lattice",
"num-bigint 0.5.1",
"num-bigint 0.4.8",
"p256 0.14.0",
"p384 0.14.0",
"p521",
@@ -9948,11 +9978,11 @@ dependencies = [
[[package]]
name = "rustfs-kafka"
version = "1.3.1"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13"
checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96"
dependencies = [
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"fnv",
"hmac 0.13.0",
@@ -9962,22 +9992,22 @@ dependencies = [
"pbkdf2 0.13.0",
"rand 0.10.2",
"rustls",
"rustls-native-certs",
"sha2 0.11.0",
"socket2",
"thiserror 2.0.20",
"tracing",
"twox-hash",
"uuid",
"webpki-roots 1.0.9",
]
[[package]]
name = "rustfs-kafka-async"
version = "1.3.1"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f"
checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630"
dependencies = [
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"hmac 0.13.0",
"kafka-protocol",
@@ -9986,11 +10016,11 @@ dependencies = [
"rand 0.10.2",
"rustfs-kafka",
"rustls",
"rustls-native-certs",
"sha2 0.11.0",
"tokio",
"tokio-rustls",
"tracing",
"uuid",
"webpki-roots 1.0.9",
]
@@ -10164,18 +10194,18 @@ dependencies = [
[[package]]
name = "rustfs-mimalloc"
version = "0.5.3"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46a7b69356718defa4060de3059e609c9da6619c35924fefef55d902ccd94958"
checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.5.3"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd3ec9b7e7b9fed453acd8b4e32713ddc0e01e1aee31a3deec5b8025c0880c3"
checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692"
dependencies = [
"cc",
]
@@ -12271,7 +12301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -12455,9 +12485,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.13.2"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b"
checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
dependencies = [
"tinyvec_macros",
]
@@ -13622,14 +13652,13 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wnaf"
version = "0.14.1"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa"
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
dependencies = [
"ff 0.14.0",
"group 0.14.0",
"hybrid-array",
"primefield",
]
[[package]]
+6 -6
View File
@@ -165,7 +165,7 @@ http-body = "1.1.0"
http-body-util = "0.1.5"
minlz = "1.2.3"
reqwest = "0.13.4"
rustfs-kafka-async = { version = "1.3.1" }
rustfs-kafka-async = { version = "1.2.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.53.1" }
tokio-rustls = { default-features = false, version = "0.26.4" }
@@ -244,8 +244,8 @@ aws-sdk-kms = { default-features = false, version = "1.117.0" }
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
aws-sdk-sts = { default-features = false, version = "1.113.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.16.0" }
aws-smithy-types = { version = "1.6.3" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
base64-simd = "0.8.0"
brotli = "9.0.0"
clap = { version = "4.6.6" }
@@ -359,15 +359,15 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "11.0.0" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.63.2" }
russh = { version = "0.63.1" }
russh-sftp = "2.4.0"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.3" }
hotpath = { version = "0.25.0", default-features = false }
rustfs-mimalloc = { version = "0.5.2" }
hotpath = { version = "0.24.0", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
-7
View File
@@ -252,13 +252,6 @@ pub mod bucket {
};
}
pub mod sealed_credentials {
pub use crate::bucket::sealed_credentials::{
CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError,
SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret,
};
}
pub mod replication {
pub use crate::bucket::replication::replication_pool::{
DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
@@ -87,12 +87,16 @@ use rustfs_filemeta::{
use rustfs_scanner_metrics::metrics::{
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
use rustfs_utils::{
get_env_i64, get_env_usize,
path::encode_dir_object,
string::{parse_bool, strings_has_prefix_fold},
};
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
Timestamp,
};
use s3s::header::X_AMZ_RESTORE;
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -161,6 +165,7 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -4797,24 +4802,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation
reader
}
/// Resolve the RestoreObject request options.
///
/// Returns the typed [`StorageError`]: flattening these into an opaque
/// `io::Error` string erased the identity the S3 layer needs to answer
/// InvalidArgument instead of a generic 500 (backlog#2205).
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, Error> {
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
let vid = version_id.trim();
if !vid.is_empty() && vid != NULL_VERSION_ID {
if let Err(_err) = Uuid::parse_str(vid) {
return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()));
return Err(std::io::Error::other(
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
));
}
if !versioned && !version_suspended {
return Err(StorageError::InvalidArgument(
bucket.to_string(),
object.to_string(),
format!("version-id specified {vid} but versioning is not enabled on {bucket}"),
return Err(std::io::Error::other(
StorageError::InvalidArgument(
bucket.to_string(),
object.to_string(),
format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
)
.to_string(),
));
}
}
@@ -4867,18 +4872,43 @@ pub async fn put_restore_opts(
}
meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/
// A SELECT restore must never reach the restore writer: the caller writes
// the retrieved bytes back to the source bucket/object, so building
// SELECT output options here produced a source overwrite carrying only
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
// is the fail-closed backstop for any other caller.
if rreq
.type_
.as_ref()
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
if let Some(type_) = &rreq.type_
&& type_.as_str() == RestoreRequestType::SELECT
{
return Err(std::io::Error::other("SELECT restore requests are not supported"));
let Some(s3) = select_restore_s3_location(rreq)? else {
return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests"));
};
if let Some(user_metadata) = s3.user_metadata.as_ref() {
for metadata in user_metadata {
let name = metadata
.name
.as_deref()
.ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?;
let value = metadata.value.clone().unwrap_or_default();
if strings_has_prefix_fold(name, "x-amz-meta") {
meta.insert(name.to_string(), value);
} else {
meta.insert(format!("x-amz-meta-{name}"), value);
}
}
}
if let Some(tags) = &s3.tagging {
meta.insert(
AMZ_OBJECT_TAGGING.to_string(),
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
);
}
if let Some(encryption) = &s3.encryption
&& encryption.encryption_type.as_str() != ""
{
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
}
return Ok(ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
user_defined: meta,
..Default::default()
});
}
for (k, v) in oi.user_defined.iter() {
meta.insert(k.to_string(), v.clone());
-1
View File
@@ -31,7 +31,6 @@ pub mod policy_sys;
pub mod quota;
pub mod remote_s3_client;
pub mod replication;
pub mod sealed_credentials;
pub mod tagging;
pub mod target;
pub mod utils;
@@ -14,9 +14,9 @@
//! Outbound client for an on-demand migration source bucket.
//!
//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The
//! S3 backend uses the shared remote builder and exposes the surface the
//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared
//! remote builder and exposes the read-only surface the migration path
//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! probe for admin validation). Every request carries the
//! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and
//! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of
@@ -511,7 +511,7 @@ pub struct SourceObject {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourcePage {
pub objects: Vec<SourceObject>,
/// Rolled-up prefixes, in the same namespace as `objects`; always empty when the
/// Rolled-up prefixes, in the local namespace; always empty when the
/// request carried no delimiter.
pub common_prefixes: Vec<String>,
pub is_truncated: bool,
@@ -519,8 +519,7 @@ pub struct SourcePage {
}
/// One `ListObjectsV2` page request against the source. Keys are given in the
/// local namespace at `SourceClient`, and in the source namespace at
/// `SourceBackend`; `SourceClient` maps them through `source_prefix`.
/// local namespace; `SourceClient` maps them through `source_prefix`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourceListRequest<'a> {
pub prefix: Option<&'a str>,
@@ -576,30 +575,8 @@ impl Intercept for SourceProxyMarkerInterceptor {
}
}
/// Read-only provider operations in the source bucket namespace.
///
/// Implementations must preserve streaming, honor the requested range and
/// pagination cursor, and classify failures without including credentials.
/// `SourceClient` owns prefix mapping so every provider shares the same local
/// namespace. Continuation tokens are opaque and must never be prefix-mapped.
#[async_trait::async_trait]
pub trait SourceBackend: Send + Sync {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError>;
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError>;
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError>;
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError>;
/// Verify bucket access; `SourceClient` separately probes a filtered listing.
async fn probe(&self) -> Result<(), SourceError>;
}
/// S3-compatible implementation, including request signing and anti-loop headers.
pub struct S3SourceBackend {
client: S3Client,
bucket: String,
}
pub struct SourceClient {
backend: Box<dyn SourceBackend>,
client: S3Client,
endpoint: String,
bucket: String,
source_prefix: Option<String>,
@@ -632,10 +609,7 @@ impl SourceClient {
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
Self {
backend: Box::new(S3SourceBackend {
client,
bucket: spec.bucket.clone(),
}),
client,
endpoint,
bucket: spec.bucket.clone(),
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
@@ -678,15 +652,35 @@ impl SourceClient {
}
pub async fn head_object(&self, key: &str) -> Result<SourceHead, SourceError> {
self.backend.head(&self.source_key(key)).await
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
}
/// Streams the object, preserving an optional HTTP byte range.
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
self.backend.get(&self.source_key(key), range).await
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
}
/// Lists one page under the local prefix.
/// Lists one page under the local `prefix`. Keys are returned in the
/// local namespace; entries outside `source_prefix` are skipped.
pub async fn list_objects_v2(
&self,
prefix: Option<&str>,
@@ -702,81 +696,9 @@ impl SourceClient {
.await
}
/// Maps keys and common prefixes while leaving opaque cursors untouched.
/// [`Self::list_objects_v2`] with the delimiter and start-after the
/// list-through merge needs (rustfs/backlog#2164).
pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
let prefix = self.source_key(request.prefix.unwrap_or_default());
let start_after = request.start_after.map(|key| self.source_key(key));
let mut page = self
.backend
.list(&SourceListRequest {
prefix: Some(&prefix),
start_after: start_after.as_deref(),
..*request
})
.await?;
page.objects = page
.objects
.into_iter()
.filter_map(|object| self.local_object(object))
.collect();
page.common_prefixes = page
.common_prefixes
.into_iter()
.filter_map(|prefix| self.local_key(&prefix).map(str::to_string))
.collect();
Ok(page)
}
fn local_object(&self, mut object: SourceObject) -> Option<SourceObject> {
object.key = self.local_key(&object.key)?.to_string();
Some(object)
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
self.backend.tagging(&self.source_key(key)).await
}
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
self.backend.probe().await?;
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
}
}
#[async_trait::async_trait]
impl SourceBackend for S3SourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
}
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
}
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
// `start_after` is silently ignored by S3 once a continuation token is
// present; refuse the ambiguous pair rather than list from the wrong
// position.
@@ -789,9 +711,9 @@ impl SourceBackend for S3SourceBackend {
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(request.prefix.unwrap_or_default())
.prefix(self.source_key(request.prefix.unwrap_or_default()))
.set_delimiter(request.delimiter.map(str::to_string))
.set_start_after(request.start_after.map(str::to_string))
.set_start_after(request.start_after.map(|after| self.source_key(after)))
.set_continuation_token(request.continuation_token.map(str::to_string))
.max_keys(request.max_keys)
.send()
@@ -809,13 +731,13 @@ impl SourceBackend for S3SourceBackend {
.contents
.unwrap_or_default()
.into_iter()
.filter_map(s3_source_object)
.filter_map(|object| self.source_object(object))
.collect();
let common_prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.filter_map(|prefix| prefix.prefix)
.filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
.collect();
Ok(SourcePage {
@@ -826,43 +748,48 @@ impl SourceBackend for S3SourceBackend {
})
}
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
fn source_object(&self, object: SdkObject) -> Option<SourceObject> {
let key = self.local_key(object.key.as_deref()?)?.to_string();
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
let output = self
.client
.get_object_tagging()
.bucket(&self.bucket)
.key(key)
.key(self.source_key(key))
.send()
.await
.map_err(classify_sdk_error)?;
Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect())
}
async fn probe(&self) -> Result<(), SourceError> {
/// Admin validation: HeadBucket plus a one-key listing under the prefix.
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
self.client
.head_bucket()
.bucket(&self.bucket)
.send()
.await
.map_err(classify_sdk_error)?;
Ok(())
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
}
}
fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
let key = object.key?;
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1267,68 +1194,6 @@ mod tests {
assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri);
}
#[tokio::test]
async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() {
let body = r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<IsTruncated>true</IsTruncated><NextContinuationToken>data/opaque</NextContinuationToken>
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
let first = client
.list_page(&SourceListRequest {
prefix: Some("photos/"),
delimiter: Some("/"),
start_after: Some("photos/a"),
max_keys: 2,
..Default::default()
})
.await
.expect("delimiter listing should succeed");
assert_eq!(first.common_prefixes, vec!["photos/"]);
assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque"));
let second = client
.list_page(&SourceListRequest {
continuation_token: first.next_continuation_token.as_deref(),
max_keys: 2,
..Default::default()
})
.await
.expect("opaque continuation should succeed");
assert_eq!(second.common_prefixes, first.common_prefixes);
let requests = recorded(&requests);
let query = |request: &RecordedRequest| {
Url::parse(&request.uri)
.expect("request URI")
.query_pairs()
.into_owned()
.collect::<HashMap<_, _>>()
};
let first_query = query(&requests[0]);
assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/"));
assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a"));
assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/"));
let second_query = query(&requests[1]);
assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque"));
assert!(!second_query.contains_key("start-after"));
}
#[tokio::test]
async fn list_page_rejects_ambiguous_cursor_before_sending() {
let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await;
let err = client
.list_page(&SourceListRequest {
start_after: Some("a"),
continuation_token: Some("opaque"),
max_keys: 1,
..Default::default()
})
.await
.expect_err("ambiguous list position must fail");
assert!(matches!(err, SourceError::Other(_)));
assert!(recorded(&requests).is_empty(), "invalid request must never reach the source");
}
#[tokio::test]
async fn list_objects_v2_rejects_truncated_page_without_token() {
let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await;
@@ -1492,14 +1357,11 @@ mod tests {
fn prefix_client(prefix: Option<String>) -> SourceClient {
SourceClient {
backend: Box::new(S3SourceBackend {
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
bucket: "bucket".to_string(),
}),
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
endpoint: "https://source.example.com".to_string(),
bucket: "bucket".to_string(),
source_prefix: prefix.filter(|prefix| !prefix.is_empty()),
@@ -1,351 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Sealed remote credentials shared by the three stores that persist one
//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote
//! tiers (`tier-config.bin`) and on-demand migration sources
//! (`on-demand-migration.json`).
//!
//! The design record is `docs/architecture/remote-credential-sealing-adr.md`.
//! What this module owns: the versioned envelope, the encryption context that
//! binds a ciphertext to the record owning it, the sealer registration point,
//! and the fail-closed error type. What it deliberately does not own: any KMS
//! call (ECStore does not depend on `rustfs-kms`; the binary installs a
//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event
//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and
//! any decision about which stored field a consumer writes.
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, OnceLock};
/// Envelope format this build writes. A reader accepts only versions it
/// knows; an unknown version is a typed error, never a fallback.
pub const SEALED_CREDENTIAL_VERSION: u8 = 1;
/// Which store a sealed value belongs to. Part of the encryption context, so
/// a ciphertext cannot be replayed into a different store.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SealedCredentialStore {
/// `bucket-targets.json` (replication and other bucket targets).
BucketTargets,
/// `tier-config.bin` (remote tiers).
TierConfig,
/// `on-demand-migration.json` (migration sources).
OnDemandMigration,
}
impl SealedCredentialStore {
pub fn as_str(self) -> &'static str {
match self {
SealedCredentialStore::BucketTargets => "bucket-targets",
SealedCredentialStore::TierConfig => "tier-config",
SealedCredentialStore::OnDemandMigration => "on-demand-migration",
}
}
}
/// Identity of the record a secret belongs to: the store, its owner (bucket
/// name, tier name, or target ARN) and the field name. Rendered into the KMS
/// encryption context so a ciphertext moved between buckets, tiers or fields
/// fails to decrypt instead of silently authorizing a different remote.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SealScope {
pub store: SealedCredentialStore,
pub owner: String,
pub field: &'static str,
}
impl SealScope {
pub fn new(store: SealedCredentialStore, owner: impl Into<String>, field: &'static str) -> Self {
Self {
store,
owner: owner.into(),
field,
}
}
/// The encryption context handed to the sealer. Keys are stable: they are
/// part of the on-disk contract, because a ciphertext only decrypts under
/// the same context.
pub fn encryption_context(&self) -> HashMap<String, String> {
HashMap::from([
("rustfs:store".to_string(), self.store.as_str().to_string()),
("rustfs:owner".to_string(), self.owner.clone()),
("rustfs:field".to_string(), self.field.to_string()),
])
}
}
/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value
/// is not a secret, but it is noise in a log line and an operator reading one
/// should see the key it is wrapped under, not the bytes.
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct SealedCredential {
/// Envelope version; see [`SEALED_CREDENTIAL_VERSION`].
pub v: u8,
/// KMS master key id the data key is wrapped under.
pub key_id: String,
/// Master key version, when the backend reports one. Carried so the KMS
/// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell
/// stale envelopes apart; nothing here rotates on its own.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub key_version: Option<String>,
/// Algorithm label reported by the sealer, for forensics and migration.
pub alg: String,
/// Ciphertext blob as produced by the sealer, base64 (standard, padded)
/// in the JSON stores and raw inside the tier msgpack payload.
pub ct: String,
}
impl fmt::Debug for SealedCredential {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SealedCredential")
.field("v", &self.v)
.field("key_id", &self.key_id)
.field("key_version", &self.key_version)
.field("alg", &self.alg)
.field("ct", &format_args!("<{} bytes sealed>", self.ct.len()))
.finish()
}
}
impl SealedCredential {
/// Rejects an envelope this build cannot read. Called before every
/// unseal so an unknown version fails here rather than inside a backend.
pub fn check_version(&self) -> Result<(), SealedCredentialError> {
if self.v == SEALED_CREDENTIAL_VERSION {
Ok(())
} else {
Err(SealedCredentialError::UnsupportedVersion(self.v))
}
}
}
/// Why a seal or unseal did not produce a usable value. Every variant is
/// terminal for the record that carried it: a caller reports the remote as
/// unusable, and never substitutes a default or empty credential.
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum SealedCredentialError {
/// No sealer is installed: KMS is not configured, or the process has not
/// finished startup. Reading a sealed record is impossible here.
#[error("no credential sealer is installed")]
NoSealer,
/// The stored envelope is from a newer (or otherwise unknown) format.
#[error("unsupported sealed credential envelope version {0}")]
UnsupportedVersion(u8),
/// The stored bytes are not a well-formed envelope.
#[error("malformed sealed credential: {0}")]
Malformed(String),
/// The sealer refused: wrong encryption context, missing key, revoked
/// access, or a failed authentication tag.
#[error("sealed credential could not be unsealed: {0}")]
Kms(String),
}
/// The KMS-backed half, installed by the binary at startup.
#[async_trait]
pub trait CredentialSealer: Send + Sync + 'static {
/// Wraps `plaintext` under the scope's encryption context.
async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError>;
/// Unwraps a stored envelope. Must fail when the envelope was sealed
/// under a different scope.
async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError>;
}
static CREDENTIAL_SEALER: OnceLock<Arc<dyn CredentialSealer>> = OnceLock::new();
/// Installs the process-wide sealer. Returns `false` when one is already
/// installed, matching the other ECStore hooks.
pub fn install_credential_sealer(sealer: Arc<dyn CredentialSealer>) -> bool {
CREDENTIAL_SEALER.set(sealer).is_ok()
}
/// The installed sealer, or `None` when KMS is not wired. Callers that only
/// need to know whether sealing is possible use this; callers that must have
/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error.
pub fn credential_sealer() -> Option<Arc<dyn CredentialSealer>> {
CREDENTIAL_SEALER.get().cloned()
}
/// Seals one secret field. Fails closed: without a sealer the caller must
/// reject the write rather than persist the secret in clear text after the
/// operator asked for sealing.
pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
sealer.seal(plaintext, scope).await
}
/// Unseals one secret field, rejecting an unknown envelope version first.
pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
sealed.check_version()?;
let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
sealer.unseal(sealed, scope).await
}
#[cfg(test)]
mod tests {
use super::*;
use parking_lot::Mutex;
/// Stands in for the KMS-backed sealer: records the context it was called
/// with, and refuses a ciphertext presented under a different one.
#[derive(Default)]
struct FakeSealer {
sealed_contexts: Mutex<Vec<HashMap<String, String>>>,
}
#[async_trait]
impl CredentialSealer for FakeSealer {
async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
let context = scope.encryption_context();
self.sealed_contexts.lock().push(context.clone());
let mut bound = serde_json::to_string(&context).expect("context serializes");
bound.push('|');
bound.push_str(plaintext);
Ok(SealedCredential {
v: SEALED_CREDENTIAL_VERSION,
key_id: "key-1".to_string(),
key_version: Some("3".to_string()),
alg: "AES-256-GCM".to_string(),
ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()),
})
}
async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
let raw = base64_simd::STANDARD
.decode_to_vec(sealed.ct.as_bytes())
.map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes");
bound
.strip_prefix(&expected)
.and_then(|rest| rest.strip_prefix('|'))
.map(str::to_string)
.ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string()))
}
}
fn scope(owner: &str) -> SealScope {
SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key")
}
#[tokio::test]
async fn seal_round_trips_and_binds_the_scope() {
let sealer = Arc::new(FakeSealer::default());
let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal");
assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION);
assert_eq!(sealed.key_version.as_deref(), Some("3"));
assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret");
// The same ciphertext under another bucket must not unseal.
let err = sealer
.unseal(&sealed, &scope("other-bucket"))
.await
.expect_err("a ciphertext must not move between owners");
assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
// Nor under another field of the same record.
let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token");
let err = sealer
.unseal(&sealed, &other_field)
.await
.expect_err("a ciphertext must not move between fields");
assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
let contexts = sealer.sealed_contexts.lock();
assert_eq!(contexts.len(), 1);
assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration");
assert_eq!(contexts[0]["rustfs:owner"], "photos");
assert_eq!(contexts[0]["rustfs:field"], "secret_key");
}
#[tokio::test]
async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() {
let sealed = SealedCredential {
v: SEALED_CREDENTIAL_VERSION + 1,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
assert_eq!(
sealed.check_version().expect_err("a newer envelope must not be read"),
SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
);
// The global helper reports the version, not "no sealer", even in a
// process where none is installed.
assert_eq!(
unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"),
SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
);
}
#[tokio::test]
async fn without_a_sealer_both_directions_fail_closed() {
// This test binary installs no sealer, so the global helpers must
// report NoSealer rather than fall back to clear text.
assert!(credential_sealer().is_none(), "no sealer is installed in unit tests");
assert_eq!(
seal_secret("super-secret", &scope("photos")).await.expect_err("seal"),
SealedCredentialError::NoSealer
);
let sealed = SealedCredential {
v: SEALED_CREDENTIAL_VERSION,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
assert_eq!(
unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"),
SealedCredentialError::NoSealer
);
}
#[test]
fn debug_and_serde_keep_the_on_disk_shape_stable() {
let sealed = SealedCredential {
v: 1,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
// key_version is omitted when absent, so an envelope from a backend
// without version history stays compact.
assert_eq!(
serde_json::to_string(&sealed).expect("serialize"),
r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#
);
let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#)
.expect("an envelope without key_version parses");
assert_eq!(parsed, sealed);
let rendered = format!("{sealed:?}");
assert!(rendered.contains("key-1"), "{rendered}");
assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}");
}
#[test]
fn a_malformed_envelope_is_a_typed_error() {
let err = serde_json::from_str::<SealedCredential>(r#"{"v":1,"key_id":"key-1"}"#)
.map_err(|err| SealedCredentialError::Malformed(err.to_string()))
.expect_err("a truncated envelope must not parse");
assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}");
}
}
File diff suppressed because it is too large Load Diff
+9 -187
View File
@@ -18,203 +18,25 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Default, Clone)]
#[serde(default)]
pub struct TierServicePrincipalAuth {
#[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
pub tenant_id: String,
#[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
pub client_id: String,
#[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
pub client_secret: String,
}
impl TierServicePrincipalAuth {
pub(crate) fn is_empty(&self) -> bool {
self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
}
}
impl std::fmt::Debug for TierServicePrincipalAuth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TierServicePrincipalAuth")
.field("tenant_id", &self.tenant_id)
.field("client_id", &self.client_id)
.field("client_secret", &"REDACTED")
.finish()
}
}
#[derive(Serialize, Deserialize, Default, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierCreds {
#[serde(rename = "access", alias = "accessKey")]
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secret", alias = "secretKey")]
#[serde(rename = "secretKey")]
pub secret_key: String,
#[serde(rename = "awsrole", alias = "awsRole")]
#[serde(rename = "awsRole")]
pub aws_role: bool,
#[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
#[serde(rename = "awsRoleWebIdentityTokenFile")]
pub aws_role_web_identity_token_file: String,
#[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
#[serde(rename = "awsRoleArn")]
pub aws_role_arn: String,
#[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
pub azure_service_principal: TierServicePrincipalAuth,
//azsp: ServicePrincipalAuth,
#[serde(
rename = "creds",
alias = "credsJson",
alias = "credsJSON",
alias = "creds_json",
default,
skip_serializing_if = "Vec::is_empty",
with = "base64_bytes"
)]
//#[serde(rename = "credsJson")]
pub creds_json: Vec<u8>,
}
impl std::fmt::Debug for TierCreds {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TierCreds")
.field("access_key", &self.access_key)
.field("secret_key", &"REDACTED")
.field("aws_role", &self.aws_role)
.field(
"aws_role_web_identity_token_file",
&(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
)
.field("aws_role_arn", &self.aws_role_arn)
.field("azure_service_principal", &self.azure_service_principal)
.field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
.finish()
}
}
mod base64_bytes {
use super::*;
pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
}
pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum EncodedBytes {
Base64(String),
Legacy(Vec<u8>),
}
match EncodedBytes::deserialize(deserializer)? {
EncodedBytes::Base64(value) => base64_simd::STANDARD
.decode_to_vec(value.as_bytes())
.or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
.or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
.or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
.map_err(de::Error::custom),
EncodedBytes::Legacy(value) => Ok(value),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
let creds: TierCreds = serde_json::from_value(serde_json::json!({
"access": "access",
"secret": "secret",
"awsrole": false,
"creds": encoded,
}))
.expect("madmin tier credentials should decode");
assert_eq!(creds.access_key, "access");
assert_eq!(creds.secret_key, "secret");
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
assert_eq!(wire["access"], "access");
assert_eq!(wire["secret"], "secret");
assert_eq!(wire["creds"], encoded);
assert!(wire.get("accessKey").is_none());
assert!(wire.get("secretKey").is_none());
let legacy: TierCreds = serde_json::from_value(serde_json::json!({
"accessKey": "legacy-access",
"secretKey": "legacy-secret",
"credsJson": service_account,
}))
.expect("the former RustFS field names and byte-array encoding should remain readable");
assert_eq!(legacy.access_key, "legacy-access");
assert_eq!(legacy.secret_key, "legacy-secret");
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
}
#[test]
fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
for encoder in [
base64_simd::STANDARD,
base64_simd::STANDARD_NO_PAD,
base64_simd::URL_SAFE,
base64_simd::URL_SAFE_NO_PAD,
] {
let encoded = encoder.encode_to_string(service_account);
let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
.expect("all supported madmin base64 forms should decode");
assert_eq!(creds.creds_json, service_account);
}
}
#[test]
fn tier_creds_debug_redacts_secret_payloads() {
let creds = TierCreds {
access_key: "access".to_string(),
secret_key: "tier-secret-value".to_string(),
aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
..Default::default()
};
let rendered = format!("{creds:?}");
assert!(!rendered.contains("tier-secret-value"));
assert!(!rendered.contains("/var/run/private-token"));
assert!(!rendered.contains("gcs-private-key-value"));
}
#[test]
fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
let creds: TierCreds = serde_json::from_value(serde_json::json!({
"azSP": {
"TenantID": "tenant",
"ClientID": "client",
"ClientSecret": "service-principal-secret"
}
}))
.expect("canonical madmin azure service principal credentials should decode");
assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
assert_eq!(creds.azure_service_principal.client_id, "client");
assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
assert_eq!(wire["azSP"]["TenantID"], "tenant");
assert_eq!(wire["azSP"]["ClientID"], "client");
assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
assert!(!format!("{creds:?}").contains("service-principal-secret"));
}
}
+123 -295
View File
@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
pub enum TierType {
#[default]
Unsupported,
#[serde(rename = "s3", alias = "S3")]
#[serde(rename = "s3")]
S3,
#[serde(rename = "wasabi")]
Wasabi,
@@ -58,7 +58,7 @@ pub enum TierType {
Huaweicloud,
#[serde(rename = "azure")]
Azure,
#[serde(rename = "gcs", alias = "GCS")]
#[serde(rename = "gcs")]
GCS,
#[serde(rename = "r2")]
R2,
@@ -138,18 +138,16 @@ impl TierType {
}
}
pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
#[derive(Default, Serialize, Deserialize)]
#[derive(Default, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct TierConfig {
#[serde(skip)]
pub version: String,
#[serde(rename = "type", alias = "Type")]
#[serde(rename = "type")]
pub tier_type: TierType,
#[serde(rename = "Name", alias = "name", skip_serializing)]
#[serde(skip)]
pub name: String,
#[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
pub s3: Option<TierS3>,
#[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
pub wasabi: Option<TierWasabi>,
@@ -161,7 +159,7 @@ pub struct TierConfig {
pub huaweicloud: Option<TierHuaweicloud>,
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
pub azure: Option<TierAzure>,
#[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
#[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
pub gcs: Option<TierGCS>,
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
pub r2: Option<TierR2>,
@@ -172,91 +170,109 @@ pub struct TierConfig {
}
impl Clone for TierConfig {
fn clone(&self) -> Self {
self.redacted()
fn clone(&self) -> TierConfig {
let mut s3 = None;
let mut wasabi = None;
let mut r = None;
let mut compatible_backend = None;
let mut aliyun = None;
let mut tencent = None;
let mut huaweicloud = None;
let mut azure = None;
let mut gcs = None;
let mut r2 = None;
match self.tier_type {
TierType::S3 => {
if let Some(s3_) = self.s3.as_ref() {
let mut s3_clone = s3_.clone();
s3_clone.secret_key = "REDACTED".to_string();
s3 = Some(s3_clone);
}
}
TierType::Wasabi => {
if let Some(wasabi_) = self.wasabi.as_ref() {
let mut wasabi_clone = wasabi_.clone();
wasabi_clone.secret_key = "REDACTED".to_string();
wasabi = Some(wasabi_clone);
}
}
TierType::RustFS => {
if let Some(r_) = self.rustfs.as_ref() {
let mut r_clone = r_.clone();
r_clone.secret_key = "REDACTED".to_string();
r = Some(r_clone);
}
}
TierType::MinIO => {
if let Some(compatible_backend_) = self.minio.as_ref() {
let mut compatible_backend_clone = compatible_backend_.clone();
compatible_backend_clone.secret_key = "REDACTED".to_string();
compatible_backend = Some(compatible_backend_clone);
}
}
TierType::Aliyun => {
if let Some(aliyun_) = self.aliyun.as_ref() {
let mut aliyun_clone = aliyun_.clone();
aliyun_clone.secret_key = "REDACTED".to_string();
aliyun = Some(aliyun_clone);
}
}
TierType::Tencent => {
if let Some(tencent_) = self.tencent.as_ref() {
let mut tencent_clone = tencent_.clone();
tencent_clone.secret_key = "REDACTED".to_string();
tencent = Some(tencent_clone);
}
}
TierType::Huaweicloud => {
if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
let mut huaweicloud_clone = huaweicloud_.clone();
huaweicloud_clone.secret_key = "REDACTED".to_string();
huaweicloud = Some(huaweicloud_clone);
}
}
TierType::Azure => {
if let Some(azure_) = self.azure.as_ref() {
let mut azure_clone = azure_.clone();
azure_clone.secret_key = "REDACTED".to_string();
azure = Some(azure_clone);
}
}
TierType::GCS => {
if let Some(gcs_) = self.gcs.as_ref() {
let mut gcs_clone = gcs_.clone();
gcs_clone.creds = "REDACTED".to_string();
gcs = Some(gcs_clone);
}
}
TierType::R2 => {
if let Some(r2_) = self.r2.as_ref() {
let mut r2_clone = r2_.clone();
r2_clone.secret_key = "REDACTED".to_string();
r2 = Some(r2_clone);
}
}
_ => (),
}
TierConfig {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
s3,
wasabi,
rustfs: r,
minio: compatible_backend,
aliyun,
tencent,
huaweicloud,
azure,
gcs,
r2,
}
}
}
impl TierConfig {
pub(crate) fn redacted(&self) -> Self {
let mut redacted = Self {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
..Default::default()
};
match self.tier_type {
TierType::S3 => {
redacted.s3 = self.s3.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
if !backend.aws_role_web_identity_token_file.is_empty() {
backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
}
backend
});
}
TierType::Wasabi => {
redacted.wasabi = self.wasabi.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::RustFS => {
redacted.rustfs = self.rustfs.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::MinIO => {
redacted.minio = self.minio.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Aliyun => {
redacted.aliyun = self.aliyun.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Tencent => {
redacted.tencent = self.tencent.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Huaweicloud => {
redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Azure => {
redacted.azure = self.azure.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
if !backend.sp_auth.client_secret.is_empty() {
backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
}
backend
});
}
TierType::GCS => {
redacted.gcs = self.gcs.clone().map(|mut backend| {
backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::R2 => {
redacted.r2 = self.r2.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Unsupported => {}
}
redacted
}
pub(crate) fn clone_with_credentials(&self) -> Self {
Self {
version: self.version.clone(),
@@ -356,61 +372,31 @@ impl TierConfig {
}
}
impl std::fmt::Debug for TierConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let redacted = self.redacted();
f.debug_struct("TierConfig")
.field("version", &redacted.version)
.field("tier_type", &redacted.tier_type)
.field("name", &redacted.name)
.field("s3", &redacted.s3)
.field("wasabi", &redacted.wasabi)
.field("aliyun", &redacted.aliyun)
.field("tencent", &redacted.tencent)
.field("huaweicloud", &redacted.huaweicloud)
.field("azure", &redacted.azure)
.field("gcs", &redacted.gcs)
.field("r2", &redacted.r2)
.field("rustfs", &redacted.rustfs)
.field("minio", &redacted.minio)
.finish()
}
}
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierS3 {
#[serde(alias = "Name")]
pub name: String,
#[serde(alias = "Endpoint")]
pub endpoint: String,
#[serde(rename = "accessKey", alias = "AccessKey")]
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey", alias = "SecretKey")]
#[serde(rename = "secretKey")]
pub secret_key: String,
#[serde(alias = "Bucket")]
pub bucket: String,
#[serde(alias = "Prefix")]
pub prefix: String,
#[serde(alias = "Region")]
pub region: String,
#[serde(rename = "storageClass", alias = "StorageClass")]
#[serde(rename = "storageClass")]
pub storage_class: String,
#[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
#[serde(skip)]
pub aws_role: bool,
#[serde(
rename = "AWSRoleWebIdentityTokenFile",
alias = "awsRoleWebIdentityTokenFile",
skip_serializing
)]
#[serde(skip)]
pub aws_role_web_identity_token_file: String,
#[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
#[serde(skip)]
pub aws_role_arn: String,
#[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
#[serde(skip)]
pub aws_role_session_name: String,
#[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
#[serde(skip)]
pub aws_role_duration_seconds: i32,
}
@@ -637,11 +623,8 @@ pub struct TierHuaweicloud {
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct ServicePrincipalAuth {
#[serde(alias = "TenantID")]
pub tenant_id: String,
#[serde(alias = "ClientID")]
pub client_id: String,
#[serde(alias = "ClientSecret")]
pub client_secret: String,
}
@@ -657,9 +640,9 @@ pub struct TierAzure {
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageClass", alias = "StorageClass")]
#[serde(rename = "storageClass")]
pub storage_class: String,
#[serde(rename = "spAuth", alias = "SPAuth")]
#[serde(rename = "spAuth")]
pub sp_auth: ServicePrincipalAuth,
}
@@ -713,19 +696,14 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierGCS {
#[serde(alias = "Name")]
pub name: String,
#[serde(alias = "Endpoint")]
pub endpoint: String,
#[serde(rename = "creds", alias = "Creds")]
#[serde(rename = "creds")]
pub creds: String,
#[serde(alias = "Bucket")]
pub bucket: String,
#[serde(alias = "Prefix")]
pub prefix: String,
#[serde(alias = "Region")]
pub region: String,
#[serde(rename = "storageClass", alias = "StorageClass")]
#[serde(rename = "storageClass")]
pub storage_class: String,
}
@@ -747,43 +725,6 @@ pub struct TierR2 {
mod tests {
use super::*;
#[test]
fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
assert!(matches!(s3, TierType::S3));
assert!(matches!(gcs, TierType::GCS));
assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
}
#[test]
fn azure_service_principal_accepts_canonical_madmin_field_names() {
for field in ["TenantID", "ClientID", "ClientSecret"] {
let mut sp_auth = serde_json::Map::new();
sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
let config: TierConfig = serde_json::from_value(serde_json::json!({
"type": "azure",
"Name": "COLD-AZURE",
"azure": {
"name": "COLD-AZURE",
"endpoint": "https://azure.example.invalid",
"accessKey": "account",
"secretKey": "key",
"bucket": "archive",
"SPAuth": sp_auth
}
}))
.expect("mixed RustFS/madmin Azure payload should decode");
let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
assert!(
!sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
"canonical {field} must not be silently discarded"
);
}
}
fn wasabi_config() -> TierWasabi {
TierWasabi {
name: "COLD-WASABI".to_string(),
@@ -897,14 +838,9 @@ mod tests {
let config = TierConfig {
tier_type: TierType::Wasabi,
wasabi: Some(wasabi_config()),
rustfs: Some(TierRustFS {
access_key: "inactive-access".to_string(),
secret_key: "inactive-secret".to_string(),
..Default::default()
}),
..Default::default()
};
let redacted = config.redacted();
let redacted = config.clone();
assert_eq!(
redacted
.wasabi
@@ -913,46 +849,21 @@ mod tests {
.secret_key,
"REDACTED"
);
assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
let cloned = config.clone();
assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
let preserved = config.clone_with_credentials();
assert_eq!(
preserved
config
.clone_with_credentials()
.wasabi
.as_ref()
.expect("credential-bearing Wasabi snapshot should remain")
.expect("credential-bearing Wasabi payload should remain")
.secret_key,
"secret"
);
assert_eq!(
preserved
.rustfs
.expect("credential-bearing snapshots should preserve inactive provider data")
.secret_key,
"inactive-secret"
);
let mut debug_config = wasabi_config();
debug_config.secret_key = "wasabi-debug-secret-value".to_string();
let debug = format!("{debug_config:?}");
assert!(debug.contains("REDACTED"));
assert!(!debug.contains("wasabi-debug-secret-value"));
let debug = format!(
"{:?}",
TierConfig {
tier_type: TierType::RustFS,
rustfs: Some(TierRustFS {
secret_key: "rustfs-debug-secret-value".to_string(),
..Default::default()
}),
..Default::default()
}
);
assert!(debug.contains("REDACTED"));
assert!(!debug.contains("rustfs-debug-secret-value"));
}
#[test]
@@ -983,7 +894,7 @@ mod tests {
assert_eq!(encoded, expected);
let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
assert!(matches!(decoded.tier_type, TierType::Wasabi));
let redacted = config.redacted();
let redacted = config.clone();
assert_eq!(
config
.wasabi
@@ -1005,87 +916,4 @@ mod tests {
"REDACTED"
);
}
#[test]
fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
let cases = [
(
"s3",
TierConfig {
tier_type: TierType::S3,
s3: Some(TierS3 {
secret_key: "s3-secret-bytes".to_string(),
aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
..Default::default()
}),
..Default::default()
},
vec!["s3-secret-bytes", "/var/run/s3-private-token"],
),
(
"gcs",
TierConfig {
tier_type: TierType::GCS,
gcs: Some(TierGCS {
creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
..Default::default()
}),
..Default::default()
},
vec!["gcs-private-key-bytes"],
),
(
"azure",
TierConfig {
tier_type: TierType::Azure,
azure: Some(TierAzure {
secret_key: "azure-account-secret-bytes".to_string(),
sp_auth: ServicePrincipalAuth {
client_secret: "azure-client-secret-bytes".to_string(),
..Default::default()
},
..Default::default()
}),
..Default::default()
},
vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
),
];
for (provider, config, secrets) in cases {
let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
let debug = format!("{config:?}");
assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
assert!(
debug.contains(TIER_CREDENTIAL_REDACTED),
"{provider} Debug output should be visibly redacted"
);
for secret in secrets {
assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
}
}
}
#[test]
fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
let config = TierConfig {
tier_type: TierType::Azure,
azure: Some(TierAzure {
secret_key: "azure-account-secret-bytes".to_string(),
sp_auth: ServicePrincipalAuth::default(),
..Default::default()
}),
..Default::default()
};
let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
let debug = format!("{config:?}");
assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
assert!(debug.contains("client_secret: \"\""));
assert!(!debug.contains("client_secret: \"REDACTED\""));
assert!(!debug.contains("azure-account-secret-bytes"));
}
}
+32 -565
View File
@@ -20,7 +20,7 @@
use crate::error::is_err_bucket_not_found;
use crate::services::tier::{
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
warm_backend_aliyun::WarmBackendAliyun,
@@ -55,21 +55,18 @@ use s3s::header::{
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use time::OffsetDateTime;
use time::format_description::well_known::{Rfc2822, Rfc3339};
use tokio::io::AsyncReadExt;
use tracing::{info, warn};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
const PROBE_OBJECT: &str = "probeobject";
/// Largest object the S3-compatible warm backends accept for a multipart put.
pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
/// Part-count ceiling S3-compatible services impose on a multipart upload.
pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);
#[derive(Default)]
pub struct WarmBackendGetOpts {
@@ -263,23 +260,6 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
}
/// Return the authority format accepted by `TransitionClient::new` while
/// retaining an explicitly configured port. `url::Url::host_str()` omits the
/// brackets needed when an IPv6 literal is combined with a port.
pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
let host = url
.host_str()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
if host.starts_with('[') && host.ends_with(']') {
Ok(format!("{host}:{port}"))
} else if host.contains(':') {
Ok(format!("[{host}]:{port}"))
} else {
Ok(format!("{host}:{port}"))
}
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
///
/// Credential, bucket, and endpoint validation run in this order because the
@@ -318,11 +298,17 @@ pub(crate) async fn new_s3_compatible_warm_backend(
bucket_lookup: params.bucket_lookup,
..Default::default()
};
let endpoint = endpoint_authority(&u)?;
// Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text.
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let host = u
.host_str()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
// Runs after the host-presence check above (not immediately after Url::parse) so a
// host-less endpoint still reports this constructor's own "missing host" text instead of
// validate_endpoint's differently-worded rejection for the same input.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
let client =
TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -465,187 +451,25 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
}
}
async fn remove_discovered_probe_candidate(
w: &WarmBackendImpl,
probe_object: &str,
candidate: TransitionCandidateProbe,
) -> Result<bool, std::io::Error> {
match candidate {
TransitionCandidateProbe::Missing => Ok(false),
TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
w.remove_exact(probe_object, &remote_version_id).await?;
Ok(true)
}
TransitionCandidateProbe::UnversionedPresent => {
w.remove(probe_object, "").await?;
Ok(true)
}
TransitionCandidateProbe::Ambiguous => {
Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
}
TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot discover the outcome of a probe PUT",
)),
}
}
async fn compensate_uncertain_probe_put(
w: &WarmBackendImpl,
probe_object: &str,
settle_deadline: tokio::time::Instant,
) -> Result<(), std::io::Error> {
let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
let mut removed_any = false;
while tokio::time::Instant::now() < settle_deadline {
let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
Ok(candidate) => candidate?,
Err(_) => break,
};
if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
break;
}
removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
let now = tokio::time::Instant::now();
if now >= settle_deadline {
break;
}
tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
}
let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
.await
.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
})??;
if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
.await
.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
})??
{
return Ok(());
}
let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
match final_candidate {
TransitionCandidateProbe::Missing => Ok(()),
_ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
}
}
fn probe_cleanup_incomplete_error() -> AdminError {
let mut err = ERR_TIER_PERM_ERR.clone();
err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
err
}
async fn check_warm_backend_with_deadlines(
w: Option<&WarmBackendImpl>,
deadline: tokio::time::Instant,
cleanup_deadline: tokio::time::Instant,
) -> Result<(), AdminError> {
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
let timeout_error = || {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out validating the remote tier mutation".to_string();
err
};
tokio::time::timeout_at(deadline, w.validate())
w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
let remote_version_id = w
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
.await
.map_err(|_| timeout_error())?
.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
let put_result =
tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
let remote_version_id = match put_result {
Ok(Ok(remote_version_id)) => remote_version_id,
Ok(Err(_)) => {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_PERM_ERR.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
Err(_) => {
let err = timeout_error();
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => err,
Err(_) => probe_cleanup_incomplete_error(),
});
}
};
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
// S3-family backends do not replay a failed request before returning `Ok`,
// while GCS discovers every matching generation. The authoritative probe
// below therefore closes the acknowledged-PUT path; only an error or
// timeout needs the longer visibility reconciliation above.
let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
Ok(Ok(candidate)) => candidate,
Ok(Err(_)) | Err(_) => {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
};
let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
let response_matches_candidate = match &authoritative_candidate {
TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
};
if !response_version_is_valid || !response_matches_candidate {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
let read_result = tokio::time::timeout_at(deadline, async {
let mut reader = w
.get(
&probe_object,
&remote_version_id,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
if w.validate_remote_version_id(&remote_version_id).is_err() {
w.remove_exact(PROBE_OBJECT, &remote_version_id)
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
let mut body = Vec::new();
reader
.take(7)
.read_to_end(&mut body)
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
if body != b"RustFS" {
return Err(ERR_TIER_PERM_ERR.clone());
}
Ok(())
})
.await
.map_err(|_| timeout_error())
.and_then(|result| result);
let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
}
match w.probe_transition_candidate(&probe_object).await? {
TransitionCandidateProbe::Missing => Ok(()),
_ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
}
})
.await;
if !matches!(cleanup_result, Ok(Ok(()))) {
return Err(probe_cleanup_incomplete_error());
return Err(ERR_TIER_INVALID_CONFIG.clone());
}
if let Err(err) = read_result {
let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
//xhttp.DrainBody(r);
if read_result.is_err() || remove_result.is_err() {
//if is_err_bucket_not_found(&err) {
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
//}
@@ -653,28 +477,12 @@ async fn check_warm_backend_with_deadlines(
return Err(ERR_TIER_MISSING_CREDENTIALS);
}*/
//else {
return Err(err);
return Err(ERR_TIER_PERM_ERR.clone());
//}
}
Ok(())
}
/// Validate a backend using a caller-owned deadline while retaining a bounded
/// reconciliation window for an uncertain probe PUT. The validation future is
/// kept alive through cleanup so an outer timeout cannot abandon the remote
/// probe object.
pub(crate) async fn check_warm_backend_until(
w: Option<&WarmBackendImpl>,
deadline: tokio::time::Instant,
) -> Result<(), AdminError> {
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
}
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
}
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
let mut d: Option<WarmBackendImpl> = None;
match tier.tier_type {
@@ -893,7 +701,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
if probe {
check_warm_backend(Some(&d)).await?;
d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
}
Ok(d)
}
@@ -946,7 +754,6 @@ pub(crate) async fn new_transition_candidate_reconciler(
#[cfg(test)]
mod tests {
use super::*;
use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
use crate::services::tier::tier_config::TierWasabi;
use std::sync::{
Arc,
@@ -1113,38 +920,13 @@ mod tests {
struct RejectingProbeVersionBackend {
gets: Arc<AtomicUsize>,
present: Arc<std::sync::atomic::AtomicBool>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
returned_version: String,
}
struct RecordingProbeBackend {
get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
present: Arc<std::sync::atomic::AtomicBool>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
remove_clears_candidate: bool,
fail_get: bool,
body: ProbeBody,
}
struct HangingProbePutBackend {
put_started: Arc<tokio::sync::Notify>,
present: Arc<std::sync::atomic::AtomicBool>,
probes: Arc<AtomicUsize>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
}
struct LateVisibleProbeBackend {
visible_at: tokio::time::Instant,
removed: Arc<std::sync::atomic::AtomicBool>,
probes: Arc<AtomicUsize>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
}
#[derive(Clone, Copy)]
enum ProbeBody {
Exact,
Mismatch,
}
#[async_trait::async_trait]
@@ -1194,7 +976,7 @@ mod tests {
}
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
Ok(self.returned_version.clone())
Ok(uuid::Uuid::nil().to_string())
}
async fn put_with_meta(
@@ -1217,19 +999,10 @@ mod tests {
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.present.store(false, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
@@ -1256,121 +1029,15 @@ mod tests {
if self.fail_get {
Err(std::io::Error::other("probe GET failed"))
} else {
match self.body {
ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
}
Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
}
}
async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
if self.remove_clears_candidate {
self.present.store(false, Ordering::SeqCst);
}
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
}
#[async_trait::async_trait]
impl WarmBackend for HangingProbePutBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
self.put_started.notify_one();
std::future::pending().await
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
}
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.present.store(false, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probes.fetch_add(1, Ordering::SeqCst);
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
}
#[async_trait::async_trait]
impl WarmBackend for LateVisibleProbeBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"probe PUT response was lost before the object became visible",
))
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
}
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.removed.store(true, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probes.fetch_add(1, Ordering::SeqCst);
if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
@@ -1431,15 +1098,13 @@ mod tests {
assert_eq!(probe, TransitionCandidateProbe::Unsupported);
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
let gets = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
gets: gets.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
returned_version: uuid::Uuid::nil().to_string(),
});
let err = check_warm_backend(Some(&backend))
@@ -1448,27 +1113,7 @@ mod tests {
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(gets.load(Ordering::SeqCst), 0);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
let gets = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
gets: gets.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
returned_version: String::new(),
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("an empty PUT version must not read or generically delete a versioned object");
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(gets.load(Ordering::SeqCst), 0);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
}
#[tokio::test]
@@ -1477,11 +1122,8 @@ mod tests {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: get_versions.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: false,
body: ProbeBody::Exact,
});
check_warm_backend(Some(&backend))
@@ -1498,11 +1140,8 @@ mod tests {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: get_versions.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: true,
body: ProbeBody::Exact,
});
let err = check_warm_backend(Some(&backend))
@@ -1514,169 +1153,6 @@ mod tests {
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_removes_probe_after_body_mismatch() {
let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions,
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: false,
body: ProbeBody::Mismatch,
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("a mismatched body should fail after cleanup");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: false,
fail_get: false,
body: ProbeBody::Exact,
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("cleanup must not succeed while the deleted candidate remains visible");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_reconciles_a_lost_put_response() {
let backend = MockWarmBackend::new();
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("a lost probe PUT response must fail after compensation");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert_eq!(backend.object_count().await, 0);
assert_eq!(backend.exact_remove_count(), 1);
let operations = backend.op_log().await;
let put = operations.iter().find_map(|operation| match operation {
MockWarmOp::Put { object } => Some(object),
_ => None,
});
let probe = operations.iter().find_map(|operation| match operation {
MockWarmOp::Probe { object } => Some(object),
_ => None,
});
let remove = operations.iter().find_map(|operation| match operation {
MockWarmOp::Remove { object } => Some(object),
_ => None,
});
let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
panic!("lost-response compensation should PUT, probe, and remove");
};
assert_eq!(put, probe);
assert_eq!(probe, remove);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
let probes = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
probes: probes.clone(),
removed_versions: removed_versions.clone(),
});
let err = check_warm_backend(Some(&driver))
.await
.expect_err("a late-visible probe PUT must still report the lost response");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(
probes.load(Ordering::SeqCst) > 5,
"reconciliation must not stop at the first Missing result"
);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
let backend = MockWarmBackend::new();
backend.set_transition_candidate_probe_override(Some(candidate)).await;
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("an uncertain candidate must fail without a guessed delete");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(backend.remove_count().await, 0);
assert_eq!(backend.object_count().await, 1);
}
}
#[tokio::test]
async fn check_warm_backend_reports_an_exact_cleanup_failure() {
let backend = MockWarmBackend::new();
backend.set_remove_failure(true);
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("an exact cleanup failure must replace the ambiguous PUT error");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.object_count().await, 1);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_reconciles_a_timed_out_put() {
let put_started = Arc::new(tokio::sync::Notify::new());
let probes = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
put_started: put_started.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
probes: probes.clone(),
removed_versions: removed_versions.clone(),
});
let check = check_warm_backend(Some(&driver));
tokio::pin!(check);
tokio::select! {
_ = put_started.notified() => {}
result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
}
tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(
probes.load(Ordering::SeqCst) > 1,
"timed-out PUT reconciliation must keep checking through the visibility window"
);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn new_wasabi_backend_honors_probe_flag() {
let tier = TierConfig {
@@ -1822,15 +1298,6 @@ mod tests {
assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
}
#[test]
fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
assert_eq!(
endpoint_authority(&url).expect("the endpoint should have an authority"),
"[2001:db8::1]:9443"
);
}
#[tokio::test]
async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
let mut params = s3_compatible_params("http://tier.example.com:9000");
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierAliyun,
warm_backend::{
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,10 +89,6 @@ impl WarmBackend for WarmBackendAliyun {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierAzure,
warm_backend::{
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,16 +89,6 @@ impl WarmBackend for WarmBackendAzure {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
// Azure currently uses the shared S3/SigV4 transport, but its normal
// object path cannot persist exact remote versions across mixed
// RustFS releases. The mutation probe may still detect and precisely
// remove a versioned test object before rejecting that configuration.
self.0
.probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
.await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
@@ -108,23 +98,6 @@ impl WarmBackend for WarmBackendAzure {
mod tests {
use super::*;
use crate::services::tier::tier_config::TierAzure;
use rustfs_s3_client::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionClient, TransitionCore},
};
use std::sync::Arc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
}
String::from_utf8_lossy(&request).into_owned()
}
/// The SSRF guard itself is exercised once, generically, in
/// `warm_backend::tests` (see backlog#2040/backlog#2041 and
@@ -146,85 +119,4 @@ mod tests {
Err(err) => assert!(err.to_string().contains("not allowed")),
}
}
#[tokio::test]
async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
let get_request = read_request_head(&mut get_stream).await;
get_stream
.write_all(
b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
)
.await
.expect("fixture should write candidate GET response");
let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
let delete_request = read_request_head(&mut delete_stream).await;
delete_stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("fixture should write exact DELETE response");
(get_request, delete_request)
});
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"azure",
)
.await
.expect("fixture client should build"),
);
let backend = WarmBackendAzure(WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
});
assert!(
!backend.0.client.provider_version_capabilities().exact_get_delete,
"probe-only version discovery must not change Azure's persisted data-path contract"
);
let candidate = backend
.probe_transition_candidate("probe")
.await
.expect("Azure candidate should be discovered");
assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
backend
.remove_exact("probe", "azure-version")
.await
.expect("Azure candidate should be deleted by exact version");
let (get_request, delete_request) = fixture.await.expect("fixture should join");
assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
assert!(
delete_request
.lines()
.next()
.is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
);
}
}
@@ -18,13 +18,13 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::collections::HashMap;
use std::io::{Error, ErrorKind};
use std::sync::Arc;
use bytes::Bytes;
use google_cloud_auth::credentials::service_account::Builder;
use google_cloud_auth::credentials::Credentials;
use google_cloud_auth::credentials::user_account::Builder;
use google_cloud_storage as gcs;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
@@ -32,7 +32,7 @@ use std::convert::TryFrom;
use crate::services::tier::{
tier_config::TierGCS,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
@@ -43,7 +43,6 @@ use rustfs_utils::egress::validate_outbound_url;
use tracing::warn;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MAX_GCS_CANDIDATE_PAGES: usize = 64;
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
if remote_version.is_empty() {
@@ -58,85 +57,6 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
Ok(Some(generation))
}
fn append_gcs_chunk<E: std::fmt::Display>(
contents: &mut Vec<u8>,
chunk: Result<Bytes, E>,
max_response_bytes: Option<usize>,
) -> std::io::Result<()> {
let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS object response exceeded the configured byte limit",
));
}
contents.extend_from_slice(&chunk);
Ok(())
}
fn gcs_bucket_resource_name(bucket: &str) -> String {
format!("projects/_/buckets/{bucket}")
}
struct GcsCandidateObject {
name: String,
generation: i64,
}
struct GcsCandidatePage {
objects: Vec<GcsCandidateObject>,
next_page_token: String,
}
async fn probe_exact_gcs_candidate<F, Fut>(
remote_object: &str,
mut fetch_page: F,
) -> Result<TransitionCandidateProbe, std::io::Error>
where
F: FnMut(String) -> Fut,
Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
{
let mut page_token = String::new();
let mut seen_page_tokens = HashSet::new();
let mut generation = None;
let mut pages_seen = 0_usize;
loop {
pages_seen += 1;
if pages_seen > MAX_GCS_CANDIDATE_PAGES {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS candidate listing exceeded the page limit",
));
}
let response = fetch_page(page_token.clone()).await?;
for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
if candidate.generation <= 0 {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS candidate listing returned a non-positive generation",
));
}
if generation.replace(candidate.generation).is_some() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
}
if response.next_page_token.is_empty() {
break;
}
if !seen_page_tokens.insert(response.next_page_token.clone()) {
return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
}
page_token = response.next_page_token;
}
Ok(match generation {
Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
None => TransitionCandidateProbe::Missing,
})
}
pub struct WarmBackendGCS {
pub client: Arc<Storage>,
pub control: Arc<StorageControl>,
@@ -160,8 +80,8 @@ impl WarmBackendGCS {
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
}
let service_account = serde_json::from_str(&conf.creds)?;
let credentials = Builder::new(service_account)
let authorized_user = serde_json::from_str(&conf.creds)?;
let credentials = Builder::new(authorized_user)
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
//.with_backoff_policy(backoff)
.build()
@@ -178,11 +98,7 @@ impl WarmBackendGCS {
let client = Arc::new(client);
// Control-plane client: the data-plane `Storage` client cannot delete or list objects;
// delete_object/list_objects live on StorageControl.
let mut control_builder = StorageControl::builder().with_credentials(credentials);
if !conf.endpoint.is_empty() {
control_builder = control_builder.with_endpoint(conf.endpoint.clone());
}
let Ok(control) = control_builder.build().await else {
let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
return Err(std::io::Error::other("StorageControl::builder error"));
};
let control = Arc::new(control);
@@ -220,10 +136,9 @@ impl WarmBackend for WarmBackendGCS {
ReaderImpl::Body(content_body) => content_body.to_vec(),
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
};
let bucket = gcs_bucket_resource_name(&self.bucket);
let Ok(res) = Box::pin(
self.client
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
.write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
.send_buffered(),
)
.await
@@ -239,9 +154,7 @@ impl WarmBackend for WarmBackendGCS {
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let bucket = gcs_bucket_resource_name(&self.bucket);
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
let mut max_response_bytes = None;
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
@@ -257,11 +170,6 @@ impl WarmBackend for WarmBackendGCS {
.length
.try_into()
.map_err(|_| std::io::Error::other("invalid range: negative length"))?;
max_response_bytes = Some(
opts.length
.try_into()
.map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
);
req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
}
@@ -269,8 +177,8 @@ impl WarmBackend for WarmBackendGCS {
return Err(std::io::Error::other("read_object error"));
};
let mut contents = Vec::new();
while let Some(chunk) = reader.next().await {
append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
while let Ok(Some(chunk)) = reader.next().await.transpose() {
contents.extend_from_slice(&chunk);
}
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
}
@@ -282,7 +190,7 @@ impl WarmBackend for WarmBackendGCS {
let mut req = self
.control
.delete_object()
.set_bucket(gcs_bucket_resource_name(&self.bucket))
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
.set_object(self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
@@ -291,47 +199,13 @@ impl WarmBackend for WarmBackendGCS {
Ok(())
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let parent = gcs_bucket_resource_name(&self.bucket);
probe_exact_gcs_candidate(&remote_object, |page_token| {
let control = self.control.clone();
let parent = parent.clone();
let prefix = remote_object.clone();
async move {
let response = control
.list_objects()
.set_parent(parent)
.set_prefix(prefix)
.set_versions(true)
.set_page_size(2)
.set_page_token(page_token)
.send()
.await
.map_err(|err| std::io::Error::other(err.to_string()))?;
Ok(GcsCandidatePage {
objects: response
.objects
.into_iter()
.map(|candidate| GcsCandidateObject {
name: candidate.name,
generation: candidate.generation,
})
.collect(),
next_page_token: response.next_page_token,
})
}
})
.await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
// Scope the listing to this tier's prefix (matching the other warm backends) and only
// need to know whether a single object exists.
let resp = self
.control
.list_objects()
.set_parent(gcs_bucket_resource_name(&self.bucket))
.set_parent(format!("projects/_/buckets/{}", self.bucket))
.set_prefix(self.prefix.clone())
.set_page_size(1)
.send()
@@ -344,126 +218,10 @@ impl WarmBackend for WarmBackendGCS {
#[cfg(test)]
mod tests {
use super::GcsCandidateObject;
use super::GcsCandidatePage;
use super::MAX_GCS_CANDIDATE_PAGES;
use super::WarmBackendGCS;
use super::append_gcs_chunk;
use super::gcs_bucket_resource_name;
use super::parse_generation;
use super::probe_exact_gcs_candidate;
use crate::services::tier::tier_config::TierGCS;
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
use google_cloud_storage::client::{Storage, StorageControl};
use std::io::ErrorKind;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
let responses = [
format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
upload_body.len()
),
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
.to_string(),
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
.to_string(),
];
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
let mut request = Vec::new();
loop {
let mut chunk = [0_u8; 1024];
let count = stream
.read(&mut chunk)
.await
.expect("the GCS fixture should read request headers");
if count == 0 {
break;
}
request.extend_from_slice(&chunk[..count]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let header_end = request
.windows(4)
.position(|window| window == b"\r\n\r\n")
.map(|position| position + 4)
.expect("the GCS fixture should receive complete request headers");
let headers = String::from_utf8_lossy(&request[..header_end]);
if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
stream
.write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
.await
.expect("the GCS fixture should acknowledge 100-continue");
}
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
})
.unwrap_or_default();
while request.len() < header_end.saturating_add(content_length) {
let mut chunk = [0_u8; 1024];
let count = stream
.read(&mut chunk)
.await
.expect("the GCS fixture should read the request body");
if count == 0 {
break;
}
request.extend_from_slice(&chunk[..count]);
}
requests.push(String::from_utf8_lossy(&request).into_owned());
stream
.write_all(response.as_bytes())
.await
.expect("the GCS fixture should write its response");
}
requests
}
fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
GcsCandidatePage {
objects: objects
.iter()
.map(|(name, generation)| GcsCandidateObject {
name: (*name).to_string(),
generation: *generation,
})
.collect(),
next_page_token: next_page_token.to_string(),
}
}
async fn probe_candidate_pages(
remote_object: &str,
pages: Vec<GcsCandidatePage>,
) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
let mut pages = pages.into_iter();
let mut requested_tokens = Vec::new();
let result = probe_exact_gcs_candidate(remote_object, |page_token| {
requested_tokens.push(page_token);
std::future::ready(
pages
.next()
.ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
)
})
.await;
(result, requested_tokens)
}
#[test]
fn generation_parser_preserves_exact_numeric_versions() {
@@ -483,243 +241,6 @@ mod tests {
}
}
#[test]
fn body_collection_propagates_an_error_after_a_complete_prefix() {
let mut contents = Vec::new();
append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
.expect("the prefix chunk should be collected");
let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
.expect_err("a trailing stream error must not be mistaken for EOF");
assert_eq!(contents, b"RustFS");
assert!(err.to_string().contains("trailing stream failure"));
}
#[test]
fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
let mut contents = Vec::new();
let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
.expect_err("the GCS collection layer must reject an oversized probe response");
assert!(contents.is_empty());
assert_eq!(err.kind(), ErrorKind::InvalidData);
}
#[tokio::test]
async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
let (first, first_tokens) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
assert_eq!(
first.expect("an exact first-page object should be discovered"),
TransitionCandidateProbe::VersionedPresent("7".to_string())
);
assert_eq!(first_tokens, [""]);
let (later, later_tokens) = probe_candidate_pages(
"prefix/object",
vec![
candidate_page(&[("prefix/object-shadow", 8)], "next"),
candidate_page(&[("prefix/object", 9)], ""),
],
)
.await;
assert_eq!(
later.expect("an exact later-page object should be discovered"),
TransitionCandidateProbe::VersionedPresent("9".to_string())
);
assert_eq!(later_tokens, ["", "next"]);
}
#[tokio::test]
async fn candidate_probe_ignores_non_exact_prefix_matches() {
let (probe, _) = probe_candidate_pages(
"prefix/object",
vec![candidate_page(
&[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
"",
)],
)
.await;
assert_eq!(
probe.expect("prefix-only matches should not hide the exact object"),
TransitionCandidateProbe::VersionedPresent("7".to_string())
);
}
#[tokio::test]
async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
let (probe, _) = probe_candidate_pages(
"prefix/object",
vec![
candidate_page(&[("prefix/object", 7)], "next"),
candidate_page(&[("prefix/object", 8)], ""),
],
)
.await;
assert_eq!(
probe.expect("multiple exact generations should produce a conservative result"),
TransitionCandidateProbe::Ambiguous
);
}
#[tokio::test]
async fn candidate_probe_reports_missing_without_an_exact_name() {
let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
assert_eq!(
probe.expect("a complete listing without an exact name should be definitive"),
TransitionCandidateProbe::Missing
);
}
#[tokio::test]
async fn candidate_probe_rejects_non_positive_generations() {
for generation in [0, -1] {
let (probe, _) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
let err = probe.expect_err("a non-positive GCS generation must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
}
}
#[tokio::test]
async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
let (probe, requested_tokens) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
let err = probe.expect_err("a repeated GCS page token must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens, ["", "next"]);
}
#[tokio::test]
async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
let (probe, requested_tokens) = probe_candidate_pages(
"prefix/object",
vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
)
.await;
let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens, ["", "a", "b"]);
}
#[tokio::test]
async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
let pages = (0..MAX_GCS_CANDIDATE_PAGES)
.map(|index| candidate_page(&[], &format!("token-{index}")))
.collect();
let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
}
#[tokio::test]
async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("the GCS fixture should bind a loopback port");
let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
let fixture = tokio::spawn(serve_data_plane_fixture(listener));
let credentials = Anonymous::new().build();
let client = Storage::builder()
.with_endpoint(endpoint.clone())
.with_credentials(credentials.clone())
.build()
.await
.expect("the GCS data client should build");
let control = StorageControl::builder()
.with_endpoint(endpoint)
.with_credentials(credentials)
.build()
.await
.expect("the GCS control client should build");
let backend = WarmBackendGCS {
client: Arc::new(client),
control: Arc::new(control),
bucket: "tier-bucket".to_string(),
prefix: String::new(),
};
let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
let version = backend
.put(
"probe",
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
6,
)
.await
.expect("a plain configured bucket should reach the GCS upload transport");
let mut reader = backend
.get(
"probe",
&version,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
.await
.expect("a plain configured bucket should reach the GCS read transport");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("the fixture body should be readable");
let oversized_error = match backend
.get(
"probe",
&version,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
.await
{
Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
Err(err) => err,
};
let requests = fixture.await.expect("the GCS fixture task should finish");
(version, body, oversized_error.kind(), requests)
})
.await
.expect("the GCS data-plane requests should not be rejected before transport");
assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
assert_eq!(version, "123");
assert_eq!(body, b"RustFS!");
assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
assert!(
requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
"unexpected upload request line: {}",
requests[0].lines().next().unwrap_or_default()
);
assert!(
requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
"unexpected read request line: {}",
requests[1].lines().next().unwrap_or_default()
);
assert!(
requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
"the GCS probe read must preserve its seven-byte range"
);
assert!(
requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
"unexpected oversized read request line: {}",
requests[2].lines().next().unwrap_or_default()
);
assert!(
requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
"the oversized response must be fetched under the same seven-byte request boundary"
);
}
#[tokio::test]
async fn new_rejects_loopback_endpoint_before_credential_setup() {
let conf = TierGCS {
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierHuaweicloud,
warm_backend::{
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,10 +89,6 @@ impl WarmBackend for WarmBackendHuaweicloud {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
@@ -26,12 +26,11 @@ use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options, endpoint_authority,
build_transition_put_options,
},
};
use http::HeaderMap;
use rustfs_s3_client::{
api_error_response::to_error_response,
api_get_options::GetObjectOptions,
api_list::ListObjectsOptions,
api_put_object::PutObjectOptions,
@@ -44,7 +43,7 @@ use rustfs_s3_client::{
};
use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
use s3s::dto::BucketVersioningStatus;
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -75,19 +74,6 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
})
}
fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
if opts.start_offset < 0 || opts.length <= 0 {
return Ok(None);
}
usize::try_from(opts.length)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
let end_offset = opts
.start_offset
.checked_add(opts.length - 1)
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
Ok(Some((opts.start_offset, end_offset)))
}
impl WarmBackendS3 {
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -146,8 +132,10 @@ impl WarmBackendS3 {
bucket_lookup,
..Default::default()
};
let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
let host = u
.host()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -189,8 +177,10 @@ impl WarmBackendS3 {
if !rv.is_empty() {
gopts.version_id = rv.to_string();
}
if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
gopts.set_range(start_offset, end_offset)?;
if opts.start_offset >= 0 && opts.length > 0 {
gopts
.set_range(opts.start_offset, opts.start_offset + opts.length - 1)
.map_err(std::io::Error::other)?;
}
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
Ok((headers, reader))
@@ -201,62 +191,34 @@ impl WarmBackendS3 {
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
}
async fn probe_current_transition_candidate_with_header(
async fn probe_transition_candidate_versions(
&self,
object: &str,
raw_version_header: Option<&'static str>,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
match self
.get_with_headers(
object,
"",
WarmBackendGetOpts {
start_offset: 0,
length: 1,
},
)
.await
{
Ok((headers, _)) => {
let version_id = match raw_version_header {
Some(header_name) => match headers.get(header_name) {
Some(value) => {
let version_id = value.to_str().map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote object version id is not valid ASCII",
)
})?;
validate_remote_version_id(version_id)?;
Some(version_id)
}
None => None,
},
None => self.client.raw_version_id(&headers)?,
};
Ok(match version_id {
Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
None => TransitionCandidateProbe::UnversionedPresent,
})
}
Err(err) => {
let response = to_error_response(&err);
if response.code == S3ErrorCode::NoSuchKey {
Ok(TransitionCandidateProbe::Missing)
} else {
Err(err)
}
}
}
}
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
pub(crate) async fn probe_transition_candidate_with_raw_version_header(
&self,
object: &str,
raw_version_header: &'static str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
.await
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut candidates = TransitionCandidateVersions::default();
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
candidates.extend(&remote_object, &versions);
if candidates.is_ambiguous() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
if !versions.is_truncated {
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
async fn probe_transition_candidate_identity(
@@ -381,7 +343,6 @@ struct TransitionCandidateVersions {
}
impl TransitionCandidateVersions {
#[cfg(test)]
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
if self.version_id.is_some() {
@@ -392,6 +353,10 @@ impl TransitionCandidateVersions {
}
}
fn is_ambiguous(&self) -> bool {
self.ambiguous
}
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
if self.ambiguous {
return TransitionCandidateProbe::Ambiguous;
@@ -415,8 +380,6 @@ impl TransitionCandidateVersions {
mod tests {
use super::*;
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[tokio::test]
async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -435,204 +398,6 @@ mod tests {
}
}
#[tokio::test]
async fn new_preserves_an_explicit_endpoint_port() {
let conf = TierS3 {
endpoint: "https://tier.example.com:9443".to_string(),
bucket: "tier-bucket".to_string(),
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
..Default::default()
};
let backend = WarmBackendS3::new(&conf, "tier")
.await
.expect("a well-formed S3 endpoint should initialize without network I/O");
assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
assert_eq!(backend.client.endpoint_url.port(), Some(9443));
}
#[tokio::test]
async fn overflowing_get_range_is_rejected_before_network_io() {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"s3",
)
.await
.expect("fixture client should build"),
);
let backend = WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
};
let err = backend
.get_with_headers(
"probe",
"",
WarmBackendGetOpts {
start_offset: i64::MAX,
length: 2,
},
)
.await
.expect_err("an overflowing range must fail before issuing a GET");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert!(
tokio::time::timeout(Duration::from_millis(100), listener.accept())
.await
.is_err()
);
}
async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let responses = [
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
];
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
requests.push(String::from_utf8_lossy(&request).into_owned());
stream
.write_all(response.as_bytes())
.await
.expect("fixture should write candidate response");
}
requests
});
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"s3",
)
.await
.expect("fixture client should build"),
);
Some((
WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
},
fixture,
))
}
#[tokio::test]
async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
let Some((backend, fixture)) = candidate_probe_fixture().await else {
return;
};
assert_eq!(
backend
.probe_transition_candidate("versioned-probe")
.await
.expect("versioned candidate should be discovered"),
TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
);
assert_eq!(
backend
.probe_transition_candidate("unversioned-probe")
.await
.expect("unversioned candidate should be discovered"),
TransitionCandidateProbe::UnversionedPresent
);
assert_eq!(
backend
.probe_transition_candidate("missing-probe")
.await
.expect("a missing key should be classified"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_candidate("provider-missing-probe")
.await
.expect("a provider-specific missing code should be classified"),
TransitionCandidateProbe::Missing
);
let err = backend
.probe_transition_candidate("forbidden-probe")
.await
.expect_err("an authorization failure must not be mistaken for a missing key");
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
let requests = fixture.await.expect("candidate fixture should join");
for request in requests {
let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
assert!(!request.contains("?versioning"));
assert!(!request.contains("?versions"));
}
}
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
ListVersionsResult {
versions: versions
@@ -866,7 +631,8 @@ impl WarmBackend for WarmBackendS3 {
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probe_current_transition_candidate_with_header(object, None).await
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_versions(object, bucket_versioning).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierTencent,
warm_backend::{
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,10 +89,6 @@ impl WarmBackend for WarmBackendTencent {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
@@ -23,7 +23,7 @@ use uuid::Uuid;
use crate::services::tier::{
tier_config::{TierS3, TierWasabi},
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
@@ -169,10 +169,6 @@ impl WarmBackend for WarmBackendWasabi {
self.s3.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
self.s3.probe_transition_candidate(object).await
}
async fn in_use(&self) -> io::Result<bool> {
self.check_remote_bucket_unversioned().await?;
let in_use = self.s3.in_use().await?;
+2 -10
View File
@@ -10339,17 +10339,9 @@ mod tests {
}
#[cfg(feature = "test-util")]
#[test]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
run_large_stack_async_test(
"v6-checkpoint-fence-loss",
v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
);
}
#[cfg(feature = "test-util")]
async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
+1 -22
View File
@@ -57,11 +57,7 @@ fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where
D: Deserializer<'de>,
{
let code = String::deserialize(d)?;
if code == "NoSuchObject" {
return Ok(S3ErrorCode::NoSuchKey);
}
Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
}
impl Default for ErrorResponse {
@@ -329,21 +325,4 @@ mod tests {
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
}
#[test]
fn normalizes_provider_specific_missing_object_code() {
let mut headers = HeaderMap::new();
headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
let response = http_resp_to_error_response(
StatusCode::NOT_FOUND,
&headers,
b"<Error><Code>NoSuchObject</Code><Message>remote detail</Message></Error>".to_vec(),
"bucket",
"object",
);
assert_eq!(response.code, S3ErrorCode::NoSuchKey);
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
}
}
+7 -242
View File
@@ -30,9 +30,7 @@ use tokio_util::io::StreamReader;
use crate::{
api_error_response::err_invalid_argument,
api_get_options::GetObjectOptions,
transition_api::{
ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
},
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
};
use futures_util::StreamExt;
use http_body_util::BodyExt;
@@ -41,42 +39,6 @@ use hyper::body::Bytes;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use tokio_util::io::ReaderStream;
fn response_limit_from_range(opts: &GetObjectOptions) -> Result<Option<usize>, std::io::Error> {
let Some(range) = opts
.headers
.iter()
.find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str()))
else {
return Ok(None);
};
let Some((unit, bounds)) = range.split_once('=') else {
return Ok(None);
};
if !unit.eq_ignore_ascii_case("bytes") {
return Ok(None);
}
let Some((start, end)) = bounds.split_once('-') else {
return Ok(None);
};
if start.is_empty() || end.is_empty() || end.contains(',') {
return Ok(None);
}
let start = start
.parse::<u64>()
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?;
let end = end
.parse::<u64>()
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?;
let length = end
.checked_sub(start)
.and_then(|length| length.checked_add(1))
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?;
let limit = usize::try_from(length).map_err(|_| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory")
})?;
Ok(Some(limit))
}
impl TransitionClient {
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
let _ = opts;
@@ -92,7 +54,6 @@ impl TransitionClient {
object_name: &str,
opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let max_response_bytes = response_limit_from_range(opts)?;
let resp = self
.execute_method(
http::Method::GET,
@@ -120,214 +81,18 @@ impl TransitionClient {
let h = resp.headers().clone();
let mut body_vec = Vec::new();
let mut body = resp.into_body();
let body_vec = if let Some(limit) = max_response_bytes {
collect_response_body(body, limit).await?
} else {
let mut body_vec = Vec::new();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
body_vec
};
}
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
}
}
#[cfg(test)]
mod bounded_response_tests {
use super::response_limit_from_range;
use crate::{
api_get_options::GetObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
};
use http_body_util::Full;
use hyper::body::Bytes;
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
#[test]
fn closed_range_derives_a_collection_limit_without_new_public_options() {
let mut opts = GetObjectOptions::default();
opts.set_range(5, 11).expect("the closed range should be valid");
assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7));
}
#[tokio::test]
async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() {
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let max_response_bytes = response_limit_from_range(&opts)
.expect("the range should parse")
.expect("the closed range should have a limit");
let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes)
.await
.expect_err("the collection layer must reject a response larger than its limit");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let request = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let request = String::from_utf8_lossy(&request).into_owned();
let response = format!(
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream
.write_all(response.as_bytes())
.await
.expect("fixture should write response headers");
stream.write_all(body).await.expect("fixture should write response body");
request
});
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
Some((client, request))
}
#[tokio::test]
async fn real_transport_accepts_the_exact_closed_range_length() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
return;
};
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let (_, _, mut reader) = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect("a seven-byte response should fit the requested range");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("bounded response should be readable");
assert_eq!(body, b"RustFS!");
assert!(
request
.await
.expect("fixture should join")
.to_ascii_lowercase()
.contains("\r\nrange: bytes=0-6\r\n")
);
}
#[tokio::test]
async fn real_transport_rejects_a_body_larger_than_the_closed_range() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else {
return;
};
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("an eight-byte response must exceed the seven-byte range limit");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(
request
.await
.expect("fixture should join")
.to_ascii_lowercase()
.contains("\r\nrange: bytes=0-6\r\n")
);
}
#[tokio::test]
async fn overflowing_closed_range_is_rejected_before_network_io() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
let mut opts = GetObjectOptions::default();
opts.headers
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("an overflowing closed range must be rejected locally");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert!(
tokio::time::timeout(Duration::from_millis(100), listener.accept())
.await
.is_err()
);
}
}
#[derive(Default)]
pub struct GetRequest {
pub buffer: Vec<u8>,
-9
View File
@@ -105,12 +105,6 @@ struct DirtyUsageSnapshot {
covers_all_pending: bool,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct ScannerBucketScanScope {
selected_buckets: Option<Arc<HashSet<String>>>,
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
}
@@ -152,7 +146,6 @@ fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
pub struct ScannerBucketScanPlan {
buckets: Vec<BucketInfo>,
all_buckets: Arc<Vec<BucketInfo>>,
scope: ScannerBucketScanScope,
digest: DataUsageScanPlanDigest,
leader_epoch: u64,
tier_registry_generation: u64,
@@ -739,8 +732,6 @@ mod dirty_usage;
mod guards;
mod io_cache;
mod io_cycle;
#[cfg(test)]
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
pub(crate) use io_cycle::nsscanner_with_storage_status;
mod io_disk;
#[cfg(test)]
+106 -230
View File
@@ -14,93 +14,6 @@
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use super::*;
#[derive(Clone, Copy)]
pub(super) struct ScannerSetCacheGeneration {
pub(super) want_cycle: u64,
pub(super) leader_epoch: u64,
pub(super) tier_registry_generation: u64,
pub(super) source: DataUsageCacheSource,
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
}
pub(super) struct PreparedScopedSetScan {
pub(super) buckets: Vec<BucketInfo>,
pub(super) cache: DataUsageCache,
}
pub(super) fn prepare_scoped_set_scan(
old_cache: &DataUsageCache,
set_buckets: &[BucketInfo],
all_buckets: &[BucketInfo],
scope: &ScannerBucketScanScope,
generation: ScannerSetCacheGeneration,
) -> Option<PreparedScopedSetScan> {
let (Some(selected_buckets), Some(baseline_scan_plan_digest)) = (&scope.selected_buckets, scope.baseline_scan_plan_digest)
else {
return None;
};
if selected_buckets.is_empty()
|| !old_cache.info.snapshot_complete
|| old_cache.info.last_update.is_none()
|| old_cache.info.name != DATA_USAGE_ROOT
|| old_cache.info.next_cycle > generation.want_cycle
|| old_cache.info.leader_epoch != generation.leader_epoch
|| old_cache.info.tier_registry_generation != Some(generation.tier_registry_generation)
|| old_cache.info.source != Some(generation.source)
|| old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest)
|| old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
|| old_cache.checked_flatten_complete_scope(DATA_USAGE_ROOT).is_none()
{
return None;
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: generation.want_cycle,
leader_epoch: generation.leader_epoch,
tier_registry_generation: Some(generation.tier_registry_generation),
source: Some(generation.source),
snapshot_complete: false,
scan_plan_digest: Some(generation.scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
lkg_snapshot_complete: true,
lkg_next_cycle: Some(old_cache.info.next_cycle),
lkg_last_update: old_cache.info.last_update,
lkg_leader_epoch: Some(old_cache.info.leader_epoch),
lkg_scan_plan_digest: old_cache.info.scan_plan_digest,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let root_hash = crate::hash_path(DATA_USAGE_ROOT);
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
}
if selected_buckets.contains(&bucket.name) {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
continue;
}
let bucket_hash = crate::hash_path(&bucket.name);
old_cache.find(&bucket.name)?;
cache.copy_with_children(old_cache, &bucket_hash, &Some(root_hash.clone()));
cache.find(&bucket.name)?;
}
Some(PreparedScopedSetScan {
buckets: set_buckets
.iter()
.filter(|bucket| selected_buckets.contains(&bucket.name))
.cloned()
.collect(),
cache,
})
}
#[async_trait::async_trait]
impl ScannerIOCache for SetDisks {
#[tracing::instrument(skip(self, budget, scan_plan, updates))]
@@ -114,9 +27,8 @@ impl ScannerIOCache for SetDisks {
scan_mode: HealScanMode,
) -> Result<()> {
let ScannerBucketScanPlan {
mut buckets,
buckets,
all_buckets,
scope,
digest: scan_plan_digest,
leader_epoch,
tier_registry_generation,
@@ -151,57 +63,26 @@ impl ScannerIOCache for SetDisks {
"Scanner old data usage cache load failed; rebuilding from bucket caches"
);
}
let scoped_scan = prepare_scoped_set_scan(
&old_cache,
&buckets,
&all_buckets,
&scope,
ScannerSetCacheGeneration {
want_cycle,
leader_epoch,
tier_registry_generation,
source,
scan_plan_digest,
},
);
let mut scoped_cache = scoped_scan.map(|prepared| {
buckets = prepared.buckets;
prepared.cache
});
if buckets.is_empty() {
let now = SystemTime::now();
let mut cache = match scoped_cache.take() {
Some(cache) => cache,
None => {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
cache
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
last_update: Some(now),
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.info.last_update = Some(now);
cache.info.snapshot_complete = true;
cache.info.lkg_snapshot_complete = false;
cache.info.lkg_next_cycle = None;
cache.info.lkg_last_update = None;
cache.info.lkg_leader_epoch = None;
cache.info.lkg_scan_plan_digest = None;
if cache.find(DATA_USAGE_ROOT).is_none() {
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(
@@ -388,102 +269,92 @@ impl ScannerIOCache for SetDisks {
record_disk_bucket_scans_active(0, &pool_label, &set_label);
let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());
let mut cache = if let Some(cache) = scoped_cache.take() {
cache
} else {
// Fence a stale set aggregate before copying entries into per-bucket work caches.
if old_cache.info.next_cycle <= want_cycle
&& old_cache.info.leader_epoch <= leader_epoch
&& old_cache.info.tier_registry_generation != Some(tier_registry_generation)
{
old_cache.info.scan_plan_digest = None;
// Fence a stale set aggregate before copying entries into per-bucket work caches.
if old_cache.info.next_cycle <= want_cycle
&& old_cache.info.leader_epoch <= leader_epoch
&& old_cache.info.tier_registry_generation != Some(tier_registry_generation)
{
old_cache.info.scan_plan_digest = None;
}
let old_lkg = old_cache.info.snapshot_complete.then_some({
(
old_cache.info.next_cycle,
old_cache.info.last_update,
old_cache.info.leader_epoch,
old_cache.info.scan_plan_digest,
)
});
let prepare_outcome = match old_cache.prepare_for_scan(
DATA_USAGE_ROOT,
want_cycle,
leader_epoch,
source,
scan_plan_digest,
require_cache_source,
) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = want_cycle,
cached_cycle = old_cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return Ok(());
}
let old_lkg = old_cache.info.snapshot_complete.then_some({
(
old_cache.info.next_cycle,
old_cache.info.last_update,
old_cache.info.leader_epoch,
old_cache.info.scan_plan_digest,
)
});
let prepare_outcome = match old_cache.prepare_for_scan(
DATA_USAGE_ROOT,
want_cycle,
leader_epoch,
source,
scan_plan_digest,
require_cache_source,
) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_cycle = want_cycle,
cached_cycle = old_cache.info.next_cycle,
state = "stale_cycle_rejected",
"Scanner rejected a set cache cycle regression"
);
return Ok(());
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = leader_epoch,
cached_epoch = old_cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected work from an older leader epoch"
);
return Ok(());
}
outcome => outcome,
};
if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
&& let Some((cycle, last_update, epoch, digest)) = old_lkg
{
old_cache.info.lkg_snapshot_complete = true;
old_cache.info.lkg_next_cycle = Some(cycle);
old_cache.info.lkg_last_update = last_update;
old_cache.info.lkg_leader_epoch = Some(epoch);
old_cache.info.lkg_scan_plan_digest = digest;
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
cache_name = DATA_USAGE_CACHE_NAME,
requested_epoch = leader_epoch,
cached_epoch = old_cache.info.leader_epoch,
state = "stale_leader_rejected",
"Scanner rejected work from an older leader epoch"
);
return Ok(());
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
lkg_snapshot_complete: old_cache.info.lkg_snapshot_complete,
lkg_next_cycle: old_cache.info.lkg_next_cycle,
lkg_last_update: old_cache.info.lkg_last_update,
lkg_leader_epoch: old_cache.info.lkg_leader_epoch,
lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
cache
outcome => outcome,
};
if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
&& let Some((cycle, last_update, epoch, digest)) = old_lkg
{
old_cache.info.lkg_snapshot_complete = true;
old_cache.info.lkg_next_cycle = Some(cycle);
old_cache.info.lkg_last_update = last_update;
old_cache.info.lkg_leader_epoch = Some(epoch);
old_cache.info.lkg_scan_plan_digest = digest;
}
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: want_cycle,
leader_epoch,
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for bucket in all_buckets.iter() {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
let (bucket_tx, bucket_rx) = mpsc::channel::<BucketInfo>(buckets.len());
@@ -1386,6 +1257,11 @@ impl ScannerIOCache for SetDisks {
incomplete_scope.info.snapshot_complete = false;
incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete;
incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle;
incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update;
incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch;
incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest;
if let Err(e) = updates.send(incomplete_scope).await {
error!(
target: "rustfs::scanner::io",
-36
View File
@@ -63,41 +63,6 @@ pub(crate) async fn nsscanner_with_storage_status<S>(
where
S: ScannerStorage,
{
let request = ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle,
leader_epoch,
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
};
nsscanner_with_storage_status_scoped(store, request).await
}
pub(crate) struct ScannerCycleRequest {
pub(crate) ctx: CancellationToken,
pub(crate) budget: Arc<ScannerCycleBudget>,
pub(crate) updates: mpsc::Sender<DataUsageInfo>,
pub(crate) want_cycle: u64,
pub(crate) leader_epoch: u64,
pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope,
}
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
where
S: ScannerStorage,
{
let ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle,
leader_epoch,
scan_mode,
scan_scope,
} = request;
let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -315,7 +280,6 @@ where
let scan_plan = ScannerBucketScanPlan {
buckets: set_buckets,
all_buckets: Arc::clone(&all_buckets),
scope: scan_scope.clone(),
digest: scan_plan_digest,
leader_epoch,
tier_registry_generation,
-149
View File
@@ -765,155 +765,6 @@ fn bucket_usage_scan_order_prioritizes_dirty_buckets() {
assert_eq!(names, vec!["dirty", "missing", "cached"]);
}
fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsageScanPlanDigest) -> DataUsageCache {
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: 7,
last_update: Some(SystemTime::now()),
leader_epoch: 11,
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
tier_registry_generation: Some(13),
..Default::default()
},
..Default::default()
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
for (bucket, size) in buckets {
cache.replace(
bucket,
DATA_USAGE_ROOT,
DataUsageEntry {
size: *size,
objects: 1,
..Default::default()
},
);
}
cache
}
#[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let current_digest = DataUsageScanPlanDigest([2; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20), ("deleted", 30)], baseline_digest);
old_cache.replace(
"stable/prefix",
"stable",
DataUsageEntry {
size: 5,
objects: 1,
..Default::default()
},
);
let all_buckets = vec![bucket_info("stable"), bucket_info("dirty")];
let selected_buckets = Arc::new(HashSet::from(["dirty".to_string(), "deleted".to_string()]));
let prepared = prepare_scoped_set_scan(
&old_cache,
&all_buckets,
&all_buckets,
&ScannerBucketScanScope {
selected_buckets: Some(selected_buckets),
baseline_scan_plan_digest: Some(baseline_digest),
},
ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: current_digest,
},
)
.expect("complete matching set cache should support a scoped scan");
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
let stable = prepared
.cache
.checked_flatten("stable")
.expect("unselected bucket subtree should be retained");
assert_eq!((stable.size, stable.objects), (15, 2));
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
assert_eq!(prepared.cache.info.scan_plan_digest, Some(current_digest));
assert_eq!(prepared.cache.info.next_cycle, 8);
assert!(!prepared.cache.info.snapshot_complete);
assert!(prepared.cache.info.lkg_snapshot_complete);
assert_eq!(prepared.cache.info.lkg_next_cycle, Some(7));
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
let baseline_digest = DataUsageScanPlanDigest([3; 32]);
let old_cache = complete_set_usage_cache(&[("stable", 10)], baseline_digest);
let all_buckets = vec![bucket_info("stable"), bucket_info("new")];
assert!(
prepare_scoped_set_scan(
&old_cache,
&all_buckets,
&all_buckets,
&ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
},
ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([4; 32]),
},
)
.is_none()
);
}
#[test]
fn scoped_set_scan_requires_an_exact_complete_baseline() {
let baseline_digest = DataUsageScanPlanDigest([5; 32]);
let all_buckets = vec![bucket_info("dirty")];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([6; 32]),
};
let mut incomplete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
incomplete.info.snapshot_complete = false;
assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut not_durable = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
not_durable.info.last_update = None;
assert!(prepare_scoped_set_scan(&not_durable, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32]));
assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none());
let empty_scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::new())),
baseline_scan_plan_digest: Some(baseline_digest),
};
let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none());
let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
future_cache.info.next_cycle = generation.want_cycle.saturating_add(1);
assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation).is_none());
}
#[test]
fn record_set_scan_failure_preserves_first_error() {
let mut first = None;
@@ -41,7 +41,7 @@ All keys below are objects in the internal metadata bucket. The table gives the
| Protocol | Current schema/version | Canonical key | Creator and cleanup owner | Authoritative identity and mutable fields | Current durability point |
|---|---|---|---|---|---|
| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete |
| Transition transaction | `rustfs-transition-transaction-v1` | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Maximum-parity config write. Current create/update/delete calls do not use ETag preconditions |
| Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records/<aa>/<bb>/<mutation-id>.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity |
| Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators/<aa>/<bb>/<mutation-id>.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent |
| Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs/<aa>/<bb>/<job-id>.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS |
@@ -54,7 +54,6 @@ All keys below are objects in the internal metadata bucket. The table gives the
| Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/<scope-digest>.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion |
| Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts/<run-token>/<source-path>/<id-kind>/<id>.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity |
| Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup |
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
@@ -72,7 +71,7 @@ The internal config layer supplies maximum-parity writes, create-only writes, ET
| Conditional delete | Removes only the verified terminal generation; if an active decommission covers it, its terminal receipt is written first |
| Strong readback | Resolves a lost response only when key, schema, full immutable identity, state, and expected successor all match |
Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. Transition transaction v1 now uses create-only installation, exact record plus ETag read before each successor CAS, and exact ETag terminal deletion. Its `owner_epoch` and `not_after_unix_nanos` remain immutable, however, so an expired recovery worker claims only the next state generation rather than a renewable durable owner lease. The manual job's initial UUID-key write still lacks create-only installation, although all later owner/lease updates are CAS-protected.
Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. The transition transaction currently carries a fixed `owner_epoch` fence identity and a mutable `revision`, but persists with unconditional writes and deletes; those fields therefore detect some in-memory misuse but are not yet a durable exclusion fence. Changing `owner_epoch` during takeover is not current behavior and remains an open design. The manual job's initial UUID-key write has the same create-only gap, although all later owner/lease updates are CAS-protected.
### Approved target
@@ -82,7 +81,7 @@ Tier mutation intents and v6 journal/manifest records use the conditional primit
- After a timeout, connection loss, or quorum-uncertain response, the caller must strongly reread. Only the exact intended successor is success; predecessor, absence, conflict, corruption, or unavailable readback retains the record and blocks destructive action.
- A process-local mutex, cancellation token, task registry, or cached generation may reduce duplicate work but cannot authorize publication, rollback, or remote deletion.
The approved transition-transaction successor, lease/takeover fields, v1 migration, and upgrade/downgrade gates are specified in [Bounded recovery control and operator disposition](#bounded-recovery-control-and-operator-disposition). They require implementation and fleet gating before any v2 writer or destructive v1 takeover is enabled.
The exact transition-transaction lease/takeover fields and whether the existing `not_after` becomes the owner expiry are an **open design**. They must be settled with upgrade/downgrade behavior before the v1 schema changes.
## Lock and operation order
@@ -90,19 +89,18 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n
| Path | Current acquisition order | Operations allowed while held | Operations forbidden while held |
|---|---|---|---|
| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. Both guards are released for lease drain, peer Prepare, and reference proof, then reacquired in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, peer fanout, and reference proof run without either exclusive guard. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation |
| Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation |
| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition |
| v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery |
| v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version |
| v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version |
| Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes |
| Transition commit | Existing object commit locks plus exact source identity and tier-generation checks in the transition path | Publish the exact remote tuple into the matching local version | Publishing a tuple after the source identity or generation changes |
| Transition transaction cleanup | Record validation; expiry check; a next-state ETag CAS for cleanup ownership; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit renewable owner lease or bucket-lifecycle/physical-set ownership fence | Identity-bound provider probe and deletion of a known canonical candidate | The cleanup-state CAS fences a stale predecessor, but the approved target also requires an explicit recovery lease and exact all-pool source reread before DELETE |
| Recovery artifact quota admission (approved target) | Cluster-scoped recovery-admission WRITE lock first; then the one canonical control/source operation lock and any source-protocol metadata/physical locks in that protocol's existing stable order | Bounded internal artifact inventory, exact source/control validation, and create-only installation plus strong readback of one fully encoded export or disposition candidate | Acquiring admission while holding a control, source, disposition, bucket, physical, migration, or decommission guard; remote backend I/O; source-journal deletion; disposition `Applying`; releasing admission before candidate installation/readback converges |
| Transition transaction cleanup | Record validation; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit bucket-lifecycle/physical-set ownership fence or durable takeover CAS | Identity-bound provider probe and deletion of a known canonical candidate | A tier lease alone does not fence the creator. The approved target requires expired ownership, durable takeover, and exact local/source reread before DELETE |
| Manual job | Initial maximum-parity job write; then persisted bucket-level scope create/CAS; later short job/task/result metadata operations | List, checkpoint, append tasks before enqueue, append results after work, renew/take over lease | Holding metadata guards across remote transition PUT; treating the local active-job map as cluster authority. A crash between the job write and scope claim can leave `Running` without a scope record |
| Decommission receipt | Decommission coordinator's source/target record workflow; record-specific conditional writes | Copy/validate durable record, advance receipts, construct and verify expected manifest, conditionally clean exact covered source | Remote tier DELETE; deleting an uncovered or divergent source record |
Tier mutation backend validation is outside both exclusive guards and is bound to the persisted ETag snapshot. The initiating task is detached from the admin request so cancellation cannot interrupt validation cleanup. Once the coordinator Prepared record and local fence are durable, lease drain, all-node peer Prepare, and reference proof run without the tier-config namespace or `admin_updates` guard. Recovery treats an unexpired Prepared record as active even while the old config ETag remains current. Both guards are reacquired in namespace-then-admin order, and the ETag, candidate digest, exact intent identity, and expiry are revalidated immediately before config CAS. After the coordinator advances to Committed, the guards are released again for peer Commit and local publication.
The tier mutation lock scope is intentionally recorded as **current**, not ideal. Reducing it is allowed only after a durable `Prepared` intent blocks new reference creators across the fleet, existing tier-operation leases drain, and recovery can reconstruct that block without the initiating process. Which network validation can move outside the namespace lock is an **open design**.
## Transition transaction
@@ -116,7 +114,7 @@ UploadStarted -> Uploaded -> LocalCommitStarted -> Committed
\-> AbortedNoRemote
```
Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production recovery emits `CleanupPending` after an expired `Uploaded` record wins the exact successor CAS, or when an expired `UploadOutcomeUnknown` probe returns `UnversionedPresent` or `VersionedPresent` with a non-nil identifier. `LocalCommitStarted` mismatch or missing-source recovery retains the record; that cleanup edge is currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is retained and never authorizes remote deletion.
Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production code emits `CleanupPending` only when recovery probes `UploadOutcomeUnknown` as `UnversionedPresent` or as `VersionedPresent` with a non-nil identifier. The `Uploaded` abort/recovery path deletes its candidate and transaction record directly, and `LocalCommitStarted` mismatch or missing-source recovery retains the record. The `Uploaded` and `LocalCommitStarted` cleanup edges are currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is another current special case: recovery exact-deletes it and removes the record without first persisting `CleanupPending`.
The remote candidate itself is named by `canonical_transition_remote_object` under `ilm/transition-transactions/<bucket-hash>/<transaction shards>/<transaction-id>/<write-id>`. That deterministic identity is what a provider probe or exact cleanup must bind; it is distinct from the internal transaction-record key.
@@ -126,13 +124,13 @@ The creator owns the canonical remote candidate until local metadata commits the
| Observed durable state/input | Unique current owner | Current recovery decision | Approved destructive admission |
|---|---|---|---|
| `UploadStarted` | Originating transition attempt; no current recovery successor is emitted | Retain | No delete. The upload may still publish |
| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery under the exact record generation and recovery lock | Conditionally delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
| `UploadStarted` | Originating transition attempt; current durable exclusion is incomplete | Retain | No delete. The upload may still publish |
| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery, logically; current record writes do not durably exclude a concurrent worker | Delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
| `UploadOutcomeUnknown`; probe returns `UnversionedPresent` | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, delete the unversioned candidate, delete the record | Exact transaction/canonical object/backend identity, explicitly unversioned state, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` with a non-nil exact identifier | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, exact-delete that versioned candidate, delete the record | Exact transaction/canonical object/backend identity and remote version, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery retains ownership evidence | Retain | A nil identifier is invalid exact-version evidence and never becomes unversioned or remote-delete authority |
| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery | Current code directly exact-deletes that versioned candidate and deletes the record; it does not persist `CleanupPending` | This remains a versioned exact-delete candidate and must not be treated as `UnversionedPresent`. The approved target still requires durable takeover, a current tier lease, and exact reread |
| `UploadOutcomeUnknown`; probe ambiguous, unsupported, or errors | Transaction recovery retains ownership evidence | Retain | No destructive action; operator reconcile may inspect after expiry |
| `Uploaded` | Originating transition attempt until expiry; after expiry, the worker that wins `Uploaded -> CleanupPending` by exact ETag CAS | Retain while active; after expiry, persist `CleanupPending`, then recheck and delete the unreferenced candidate or record | Current CAS fences the predecessor, but approved v2 also requires a durable recovery lease, full all-pool source/free-version proof, and before/after fence checks |
| `Uploaded` | Originating transition attempt; current recovery can race it because the persisted fence is not CAS-protected | Current code immediately deletes the candidate and record | **Current safety gap:** approved behavior must first prove the creator cannot still commit by expired ownership plus durable takeover/CAS, then recheck that no matching local commit exists |
| `LocalCommitStarted`; logical source lookup returns `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Current recovery treats this tuple as ownership transfer. The approved target additionally compares recorded source version ID, data directory, modification time, size, and ETag before conditional terminal cleanup |
| `LocalCommitStarted`; logical source is missing, its transition tuple differs, or the read is uncertain | Transaction record/recovery | Retain | No remote delete without a separate durable cleanup proof |
| `CleanupPending`; logical source lookup returns the same current transition predicate | `xl.meta` is remote reachability owner; recovery owns only record cleanup | Delete transaction record | `xl.meta` is owner; do not delete remote. The approved target adds the full recorded source comparison |
@@ -150,12 +148,12 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
### Approved target and open design
- Preserve the current create-only transaction installation, exact record/ETag successor CAS, and conditional terminal delete, and add mandatory exact-successor strong readback for lost or uncertain responses. No v2 work may regress the existing v1 protections.
- Replace unconditional transaction create/update/delete with create-only, ETag CAS, and conditional terminal delete.
- Recovery of `Uploaded` and cleanup-capable states must acquire durable ownership only after the prior owner's expiry. The recovery worker must reread the exact generation after takeover and before remote DELETE.
- Preserve and compare source version ID, data directory, modification time, size, and ETag through local commit and recovery before accepting ownership transfer.
- Recompute and require the exact lowercase sharded path in both transition-transaction and manual-job runtime recovery, and validate a truncated page's continuation token before processing any record from that page.
- The approved v2 owner lease uses the duration, clock-skew allowance, takeover revision, and v1 mapping in [Transition transaction v2 and v1 migration](#transition-transaction-v2-and-v1-migration).
- Active automatic retries are bounded by the recovery-control policy below. Ambiguous evidence becomes explicit `retained_ambiguous` or `operator_required`; source evidence is never collected merely because it is old.
- **Open:** owner lease duration, clock-skew allowance, takeover revision encoding, and compatibility for existing v1 records that have only `not_after`/`owner_epoch`.
- **Open:** bounded retention and an operator disposition for permanently ambiguous records. Until defined, retained ambiguity is safer than collection.
## Tier mutation intent
@@ -165,15 +163,14 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence.
The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. The coordinator then conditionally writes tier config, commits peers, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
### Recovery decisions
| Observed durable state/input | Unique current owner | Current recovery decision | Destructive/config admission |
|---|---|---|---|
| `Prepared`; current tier-config digest equals candidate | Coordinator recovery; each peer recovery owns only its matching peer record/block | CAS to `Committed`, replay peer Commit and publish | Exact mutation identity and candidate digest; never infer from expiry |
| `Prepared`; intent is unexpired and current config still proves the old ETag/config | The live coordinator remains owner | Retain `Prepared` and the runtime block | Recovery must not abort work that may still be in lock-free peer Prepare or reference proof |
| `Prepared`; intent is expired and current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
| `Prepared`; current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
| `Prepared`; config is a third generation, unreadable, or peer outcome is ambiguous | Coordinator record remains owner of the block | Retain `Prepared` and runtime block | No commit, abort, cleanup, or unblock |
| `Committed` | Coordinator recovery, with peers owning convergence of their local records | Replay peer Commit/runtime publication; clean exact converged records | Config ETag/digest and peer identity must match |
| `Aborted` | Coordinator recovery; peer recovery retains the local tombstone | Replay/confirm Abort and clear matching block; retain peer tombstone until expiry plus clock skew and no coordinator | Never roll back config based on timeout alone |
@@ -186,8 +183,8 @@ Intent transitions retry an ETag race at most three times before returning a ret
### Approved target and open design
- Keep create-only, ETag CAS, exact identity comparison, canonical Abort tombstones, and lost-response readback.
- The phase split is fixed: ETag snapshot, lock-free backend validation, ETag revalidation, all-node Prepare, reference proof, ETag/digest/intent revalidation, config CAS, all-node Commit, and local publication. Backend validation uses `(old_config_etag, candidate_digest)` as its stable config generation; the durable mutation identity adds `mutation_id`. Runtime driver revisions are not persistence identities and Add does not require one before publication.
- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof.
- Any shorter configuration-lock window must leave a durable `Prepared` fence installed on every required peer before releasing the broad exclusion scope and must prove recovery restores that fence before admitting reference creators.
- **Open:** the exact split between backend validation, peer fanout, reference scan, config CAS, and publication; expiry must never replace config-generation proof.
- **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records.
## Manual transition job, task, result, and checkpoint
@@ -314,149 +311,7 @@ Single and child manifest preparation is bounded by 200,000 journals and a 32 Mi
- New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created.
- Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE.
- Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous.
- Quarantined v1/v2 records, incomplete manifests, and repeatedly failing exact deletes use the bounded retry, explicit operator state, and single-record control surface approved below. Capacity and recovery throughput must not be “fixed” by weakening ownership proof or age-deleting source evidence.
## Bounded recovery control and operator disposition
This section is an **approved target that is not implemented yet**. It closes the ownership, retry, and operator-disposition design required before backlog recovery work can change destructive behavior. It does not authorize an implementation to enable a v2 writer, take over a v1 transaction, or remove a legacy journal until the fleet and storage gates below exist.
### Recovery-control record
Retry scheduling is persisted separately from the source transaction or journal so legacy bytes remain readable and their cleanup ownership does not move. The control record uses schema `rustfs-ilm-recovery-control-v1`, a checksum envelope, a 16 KiB encoded-size limit, strict unknown-field rejection, and this canonical key:
```text
ilm/recovery-controls/<protocol>/<aa>/<bb>/<source-operation-digest>.json
```
Export envelopes use `ilm/recovery-exports/<protocol>/<aa>/<bb>/<export-id>.json`; disposition receipts use `ilm/recovery-dispositions/<protocol>/<aa>/<bb>/<disposition-id>.json`. `export-id` is the SHA-256 of the control ID plus exact observed source content/copy-set digests; `disposition-id` is the SHA-256 of the export ID plus the bounded action. Repeating an identical export or disposition reuses and strongly validates the same canonical object; different bytes at that ID are a conflict. Both use strict checksum envelopes and create-only installation. A control or disposition is limited to 16 KiB. An export is limited to the source protocol's own maximum encoded record size plus 64 KiB for its copy manifest and envelope; it stores the source bytes once rather than duplicating identical replica bytes.
Admission is fail-closed and cluster-wide. One control may have at most one export creation and one disposition application in flight. The immutable candidate is fully encoded before quota admission. The cluster-scoped recovery-admission WRITE lock is always outermost; a caller already holding any control, source, disposition, bucket, physical, migration, or decommission guard must release it and restart in the order above. While admission is held, a complete artifact inventory must prove that the projected totals, including the candidate, are at most 10,000 export envelopes, 10,000 disposition receipts, 1 GiB of encoded export data, and 256 MiB of encoded control/disposition data. The create-only candidate installation and exact strong readback complete before the lock is released, so neither a single candidate nor a concurrent node can oversubscribe the pre-create snapshot. A crash before installation leaves no artifact; a lost create response is resolved by exact canonical readback while admission remains serialized or after reacquiring it in the same order. Replaying an existing canonical ID consumes no new count, byte, or rate token. New creations are limited to ten per authenticated actor per minute and 100 cluster-wide per minute, with at most 32 export creations and eight disposition applications executing cluster-wide. Exceeding a count, byte, rate, or concurrency limit returns a retryable capacity result before source mutation; it never evicts evidence, interrupts an admitted operation, or blocks ordinary object I/O. The collector examines at most 100 terminal artifacts per minute and never acquires the admission lock while holding an artifact/source guard.
`source-operation-digest` is SHA-256 over the length-delimited source protocol, canonical source path, and stable semantic operation identity: transaction UUID, journal identity, or manifest operation ID. For corrupt bytes whose semantic ID cannot be trusted, the canonical path plus a `corrupt` domain separator is the stable identity; a replacement at that path conflicts with the existing control instead of resetting its history. The control's immutable identity repeats the protocol, path, stable identity, and record class. Its mutable `observed_source_generation` contains the source schema, source ETag/content SHA-256, and sorted all-pool copy-set digest. The copy set records each authoritative pool/set identity, canonical path, ETag, byte length, and content SHA-256; an unreachable pool, missing ETag, divergent copy, or incomplete listing cannot produce an actionable generation. The remaining mutable generation contains:
- `revision`, an ETag-CAS successor counter;
- `classification`: `retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`;
- `owner_id`, `owner_epoch`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos` while an attempt is owned;
- monotonic `attempt_count`, `consecutive_failure_count`, `first_failure_at_unix_nanos`, `last_failure_at_unix_nanos`, and `next_attempt_at_unix_nanos`;
- one bounded enum `last_error_code`; no free-form provider error, endpoint, credential material, request payload, or response body is persisted;
- for an operator disposition only, the authenticated actor identifier, reason code, confirmation time, exported payload digest, and exact source/control ETags that were confirmed.
The control record is a scheduler and audit fence, not a remote-object owner. It never substitutes for the source record's version, backend, manifest, source-identity, or absence proof. Before every source CAS, local cleanup, or remote request, the worker strongly rereads every authoritative source copy and the control, requires the current observed generation to match, and then applies the source protocol's own locks and proof. A missing, stale, corrupt, divergent, or unavailable control read cannot authorize work.
Control creation is `If-None-Match: *`; every update is exact ETag `If-Match`; response loss requires exact strong readback. A legal source-state CAS does not create a new control. The stable control CAS advances `observed_source_generation` only from the exact predecessor to one source-protocol successor while preserving `first_failure_at_unix_nanos`, lifetime `attempt_count`, and first-seen lineage. If the source CAS succeeded before a crash, recovery accepts the new bytes only after validating that exact legal successor and then converges the old control generation by CAS. A multi-edge jump, semantic-identity change, replacement ETag/content, or unavailable predecessor proof is a conflict and cannot reset counters. Recovery conditionally removes a terminal control only after strongly proving the stable source operation resolved or absent, recording any active decommission terminal receipt, and confirming that no in-flight operator request still names its ETag. `durable_namespace.rs` must register every new namespace, path parser, decoder, size bound, successor relation, and terminal checkpoint before rollout.
### Transition transaction v2 and v1 migration
The successor envelope is `rustfs-transition-transaction-v2`. It preserves the v1 transaction ID, deployment ID, write ID, complete source identity, tier/backend identity, canonical remote object, remote-version state, operation state, and revision. It also records an immutable `origin_format` (`native_v2` or `migrated_v1`), the state-entry revision and predecessor state/revision, and, for migration, the exact source v1 state/revision. These fields make the state history needed for destructive authorization independently checkable instead of inferring it from the current state. The envelope and body reject unknown fields, checksum every field, use no default for a required v2 value, require the existing exact lowercase canonical path, and reject nil UUIDs, nonpositive timestamps, revision zero/overflow, lease inversion, impossible state/revision history, and illegal state/version combinations. It replaces the fixed ownership pair with:
- immutable `creator_epoch` and `creator_not_after_unix_nanos`, copied exactly from v1 `owner_epoch` and `not_after_unix_nanos` during migration;
- optional `created_at_unix_nanos`; migrated v1 records use `None` rather than inventing an age;
- mutable `owner_role` (`creator` or `recovery`), `owner_id`, `owner_epoch`, `owner_generation`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos`. `owner_id` is the authenticated stable fleet-node identity; `owner_epoch` is a fresh UUID for one process claim. A node without both identities cannot create, renew, take over, or act on v2.
A native v2 create uses revision and owner generation 1, a fresh non-nil creator owner/epoch, `UploadStarted`, and an unknown remote version. The first rollout keeps the current seven-day creator ownership window. Creator leases are not renewable; a creator that cannot finish inside the safety window stops publishing and leaves the record for recovery. Recovery-owner leases are 15 minutes. The persisted clock-skew allowance is five minutes: an action may start only when its bounded deadline fits before `lease_expires_at - 5 minutes`, and another owner cannot take over until its local time is at least `lease_expires_at + 5 minutes`. A recovery attempt remains capped at five minutes and every remote call remains subject to its narrower client deadline. A recovery renewal is a same-owner, next-revision CAS that strictly extends expiry; a takeover changes `owner_role`, `owner_id`, and `owner_epoch`, increments `owner_generation` and `revision`, and uses the exact observed ETag. Takeover and state advancement are separate CAS operations; one revision cannot both acquire ownership and claim a recovery outcome.
Before migration can be enabled, the fleet must first deploy a v1 creator fence that strongly rereads the exact transaction path, ETag, state, owner epoch, and source generation immediately before local metadata publication and refuses to publish after any migration/takeover change. The fleet then durably disables new v1 admission and proves that every captured creator/recovery process epoch has either acknowledged quiescence or terminated. A paused or unreachable epoch prevents migration. This drain barrier is distinct from format capability advertisement and remains in force until v2 writer admission is enabled.
Only these checksum-valid v1 state/revision pairs are migration inputs: `UploadStarted@1`; `UploadOutcomeUnknown@2`; `AbortedNoRemote@2`; `Uploaded@2` or `Uploaded@3`; `LocalCommitStarted@3` or `LocalCommitStarted@4`; `Committed@4` or `Committed@5`; and `CleanupPending@3`, `CleanupPending@4`, or `CleanupPending@5`. Any other pair is `corrupt`, inspect-only, and cannot be migrated or authorize a probe, local cleanup, or remote DELETE. A native v2 record must prove a legal predecessor edge at its recorded state-entry revision; ownership-only revisions may increase the outer revision but cannot change the recorded state-entry history. Missing, contradictory, or skipped history is corrupt.
An identity-preserving `v1 -> v2` conversion is one legal successor with `revision + 1` and an unchanged remote tuple. The state is unchanged except that every v1 `UploadStarted` maps conservatively to v2 `UploadOutcomeUnknown`. Historical v1 writers could issue PUT while still in `UploadStarted`; no age, fleet version, or current process observation proves that a retained record came from the later pre-PUT-fence writer. It is permitted only when:
1. every node that can create, commit, recover, heal, or decommission the record advertises both `transition_transaction_v2` and `ilm_recovery_control_v1` for the captured fleet/topology generation, the durable v1-admission stop is active, and the process-epoch drain barrier above is complete;
2. current time is at least the v1 `not_after_unix_nanos` plus five minutes of skew;
3. the canonical path, checksum, full immutable identity, state, remote-version invariant, source record, and ETag all match the observed v1 generation;
4. the migration CAS and strong readback install one fresh recovery lease before any state transition or side effect.
The original creator must successfully CAS `UploadStarted -> UploadOutcomeUnknown` before issuing remote PUT, and must CAS the exact current owner generation to `LocalCommitStarted` before publishing local metadata. A v2 takeover therefore fences a delayed creator. An implementation that can issue PUT or publish after losing this CAS is not compatible with this protocol.
After takeover, recovery applies this matrix:
| State | Approved recovery after exact takeover | Required proof before side effect |
|---|---|---|
| native-v2 `UploadStarted` | CAS `AbortedNoRemote`, then conditionally remove the terminal transaction | Valid native-v2 history proves the creator was fenced before the mandatory pre-PUT `UploadOutcomeUnknown` CAS; migrated v1 never enters this row and no remote request is made |
| `UploadOutcomeUnknown` | Probe under the exact backend lease. Missing becomes terminal cleanup; proven unversioned presence or one nonempty, non-nil exact version becomes `CleanupPending`; nil, ambiguous, or unsupported results become `retained_ambiguous` | Exact transaction/control generations, bounded live probe, current tier destination and lease |
| `Uploaded` | If the exact transitioned tuple or its free-version owns the candidate, remove only the transaction. If the complete original source is still unchanged and no local commit/free-version exists, CAS `CleanupPending`. Otherwise retain | All-pool source/free-version read, full source tuple, bucket incarnation, tier generation, object locks, and post-probe revalidation |
| `LocalCommitStarted` | Exact committed tuple/free-version means record-only cleanup. A fully unchanged original source with no partial committed tuple may move to `CleanupPending`. Missing, divergent, partial, or unavailable metadata is `retained_ambiguous` | Same all-pool proof, including data directory, modification time, size, ETag, transition transaction ID, remote tuple, and destination identity |
| `CleanupPending` | Resume the same exact idempotent candidate delete, or remove only the transaction when local ownership transfer is proven | Current owner lease, source/free-version proof, exact tier lease, physical locks, and before/after fence checks |
| `Committed` | Conditionally remove only the exact terminal transaction when complete local ownership transfer is proven; otherwise classify the record as corrupt or retain it as ambiguous | All-pool strong read matches the complete logical `xl.meta` source identity, transaction ID, remote tuple, destination identity, and any required decommission terminal receipt; no remote DELETE |
| `AbortedNoRemote` | Conditionally remove the exact terminal transaction | Valid native-v2 pre-PUT history or exact migrated v1 `AbortedNoRemote@2`, exact terminal generation, and any required decommission terminal receipt; no remote DELETE |
Remote DELETE is never admitted directly from `UploadStarted`, `UploadOutcomeUnknown`, `Uploaded`, or `LocalCommitStarted`; it first requires a CAS-protected `CleanupPending` generation with known remote-version semantics. A lost source read, mixed all-pool result, expired lease, failed renewal, or source/control CAS conflict retains the evidence and performs no destructive action.
New writers emit v2 only after the homogeneous fleet gate, durable v1-admission stop, and process-epoch drain barrier are complete. During a rolling upgrade, new readers accept v1 but all writers continue v1 and no v1 takeover/migration occurs. A v1 reader rejects and retains v2. Downgrade is blocked until v2 creation is disabled and all v2 transactions and recovery-control records are drained or exported; live v2 bytes are never rewritten to v1.
### Retry and retention policy
An attempt that loses a CAS or discovers a newer source generation reloads instead of recording a remote failure. A retryable transport timeout, backend 5xx/throttle, metadata quorum outage, or bounded remote-delete failure increments the persisted counters and schedules:
```text
min(60 seconds * 2^min(consecutive_failure_count - 1, 6), 1 hour)
```
A deterministic multiplier from 80 to 100 percent, derived from the source-generation digest and attempt count, is applied to that capped base. The jitter can only shorten the delay and therefore never exceeds the one-hour cap; restarts reproduce the same deadline without synchronizing a fleet. Success or a proven source-state advance resets `consecutive_failure_count` but never decreases `attempt_count`. `next_attempt_at_unix_nanos` is only a not-before scheduler hint; ownership and destructive authority still require the lease and source proofs.
After 32 consecutive retryable failures or seven days since `first_failure_at_unix_nanos`, whichever occurs first, the control CAS moves to `operator_required` and automatic attempts stop. Unsupported probes and unknown remote-version semantics move directly to `retained_ambiguous`; corrupt source bytes use `corrupt`; incomplete destructive evidence uses `operator_required`. None is periodically hot-looped. An operator may explicitly request another bounded attempt after the underlying capability or configuration changes, but the request creates a new owner lease and preserves the lifetime attempt count.
This policy bounds automatic work, not evidence lifetime. A source transaction, journal, or manifest is never deleted solely because it is old, numerous, or over a byte threshold. `operator_required` source evidence remains until its protocol reaches a proven terminal state or the legacy-journal disposition below is completed.
Resolved control tombstones are retained for at least 30 days, immutable export envelopes for at least 90 days, and compact completed disposition receipts for at least 365 days. A collector may conditionally remove only a terminal artifact past its floor after proving that the bound source generation is absent where required, no nonterminal successor or active decommission references it, and the terminal audit checkpoint is durable. Capacity pressure blocks new export/disposition work rather than evicting unexpired or nonterminal evidence. Uncertainty retains the artifact; collection never authorizes source or remote deletion.
### Legacy journal and manifest disposition
Journal v1 has neither backend identity nor remote-version authority; v2 has backend identity but still lacks remote-version semantics. Their automatic classification is `retained_ambiguous`, and neither recovery nor an operator action may instantiate a backend or issue remote PUT, GET, probe, or DELETE from those bytes. The approved single-record actions are:
- **inspect**: strictly decode a server-reconstructed canonical journal identity, perform an all-pool strong read, and return a redacted copy-set/content digest, version, quarantine reason, control classification, age information when known, topology readiness, and decommission coverage; it changes nothing and does not return raw object/version fields by default;
- **export**: after a fresh exact inspect, create-only persist an immutable `rustfs-ilm-recovery-export-v1` envelope containing the raw source bytes and sorted copy manifest, then strongly read it back. The response downloads that envelope rather than rereading the live journal, uses no-store/attachment semantics, and never adds credentials or backend configuration;
- **abandon after export**: v1 or v2 only; create a `Prepared` `rustfs-ilm-recovery-disposition-v1` receipt bound to the immutable export and every source copy, conditionally remove only those exact local journal generations, prove every bound copy absent with no replacement, and advance the receipt through `Applying` to `Completed`. This accepts a possible remote storage leak and never asserts that cleanup occurred.
Export and abandon require a fresh all-member capability/topology proof including each member's current process epoch; inspect may remain available in a mixed fleet but returns not-ready for mutation. `abandon after export` uses POST and requires `confirm: true`, `action: abandon_remote_cleanup`, `acknowledge_remote_cleanup_abandoned: true`, the export operation ID/digest, source content and copy-set digests, every source ETag, control ETag, and a bounded operator reason code. It is refused while an active decommission or migration receipt covers either record, while physical copy discovery is incomplete, or when the implementation cannot target every discovered copy with its own `If-Match` condition.
The disposition receipt has immutable action/export/control identities and an immutable sorted copy manifest. Its ETag-CAS generation contains state `Prepared`, `Applying`, or `Completed` and a monotonic sorted `confirmed_absent` set naming only entries from that manifest. Before `Prepared -> Applying`, a fresh all-pool read must find every bound copy at its exact ETag/content digest and repeat the fleet, lock, migration, and decommission checks. The manifest can never be widened, reordered, or replaced.
During `Applying`, recovery treats each manifest entry independently while retaining the original all-pool boundary. An entry already in `confirmed_absent` must still be strongly absent with no successor or replacement generation. For an unconfirmed entry, an exact ETag/content match may be conditionally deleted and then added to `confirmed_absent` only after strong absence readback. If a crash or lost response left that exact path absent before the progress CAS, recovery may add it only after the same strong absence, stable source/control generation, topology, process-epoch, migration, and decommission proofs establish that no replacement exists. A different ETag/content, an unbound copy, an unreadable member, or loss of any proof is a conflict and preserves the current progress. Thus a crash after deleting copy A but before recording its progress can converge and continue with copy B without requiring deleted copy A to reappear.
Immediately before each local metadata deletion, the server repeats the applicable all-pool/fleet/lock checks and conditionally targets only the still-unconfirmed exact ETag. Completion requires every immutable manifest entry in `confirmed_absent`, a fresh all-pool proof that all remain absent without replacement, unchanged fleet/process epochs, and no active decommission or migration coverage. A lost final response is success only when strong readback proves the canonical receipt `Completed`. Recovery may repeat only this canonical operation and never creates a tier client or issues a backend request.
Malformed or unsupported bytes whose outer v1/v2 identity cannot be proven are inspect-only and cannot use abandon. Versions v3-v6 never use `abandon after export`. Their known candidate or manifest ownership must converge through the normal exact protocol. An operator may inspect/export and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or remote-version validation. `Preparing`/`Aborting` manifests may use their existing whole-set rollback; `DispatchAuthorized`/`Completed`, a missing member, a nonempty operation namespace, or any uncertain binding cannot be manually removed.
### Admin and metrics contract
The approved surface is single-record and uses a protocol-specific expected tuple; it does not reuse the legacy metadata-reconcile digest or create a bucket/prefix job:
```text
GET /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
GET /rustfs/admin/v3/ilm/recovery/records/<control-id>
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
GET /rustfs/admin/v3/ilm/recovery/exports/<export-id>
```
List and redacted inspect require `admin:ListTier`. Raw export creation/download, retry, or abandon require `admin:SetTier` because legacy bytes can reveal bucket, object, tier, and remote-version information. The default page is 100 records and the hard maximum is 1,000. A truncated page without a continuation marker is an error, and counts from an incomplete scan are labeled incomplete rather than reported as zero or complete.
Inspect returns a 15-minute opaque observation receipt bound to the authenticated actor, canonical record identity, source/copy ETags and digests, topology/fleet generation, every member process epoch, action class, issue/expiry time, and nonce. It is observation evidence, not mutation authority. POST requires that receipt and `confirm: true` for terminal actions, and binds the control ETag/revision/classification, requested action, and export digest when applicable. The server repeats all live proofs; a restarted member, membership change, or process-epoch mismatch invalidates the receipt. Client fields are concurrency guards, not authority. Export download reads the immutable envelope, uses TLS plus `Cache-Control: no-store` and attachment disposition, and never logs the raw payload.
Metrics use bounded labels only:
- `rustfs_ilm_recovery_records{protocol,classification,schema}` and `rustfs_ilm_recovery_oldest_age_seconds{protocol,classification,schema}`;
- `rustfs_ilm_recovery_attempts_total{protocol,outcome,error_code}`;
- `rustfs_ilm_recovery_operator_actions_total{protocol,action,outcome}`;
- scan completeness, corrupt-record, and orphan-control counters.
Every label value is a closed enum. `schema` exposes recognized schema identifiers only; an unrecognized raw value maps to `unknown`, while a recognized future schema disabled by the current fleet maps to `unsupported`. `protocol`, `classification`, `outcome`, `error_code`, and `action` likewise map unknown input to one bounded fallback and never expose decoded or operator-provided text.
Object names, bucket names, tier names, transaction IDs, control IDs, endpoints, ETags, error text, and credentials are never metric labels. Admin output may identify the selected record but redacts credentials and raw backend configuration. Audit/log events use the repository ILM event fields, stable reason codes, authenticated actor, source/control generations, action, and outcome; they do not persist or log provider response bodies.
One canonical source generation counts once regardless of its physical copy count or how many recovery passes observed it. Attempt counters increment once per coordinator attempt, not per replica, CAS retry, or page revisit. Aggregate totals and oldest age are authoritative only after a complete all-pool scan; partial coverage reports `incomplete` and never publishes a false zero. A legacy record's first-seen time comes from its durable control record rather than an inferred object modification time.
### Required protocol fixtures
Implementation acceptance requires deterministic crash/restart and mixed-version fixtures, not timing-only tests. At minimum they cover:
- a historical v1 `UploadStarted@1` whose PUT may have reached the provider, proving migration yields `UploadOutcomeUnknown` and never `AbortedNoRemote` or a direct delete;
- every accepted v1 state/revision pair above plus checksum-valid impossible pairs such as `CleanupPending@1`, proving impossible history is inspect-only and makes zero backend calls;
- an in-flight v1 creator interleaved with admission stop, process-epoch drain, migration, takeover, and local publication, proving no stale creator can publish after takeover;
- `Committed` with complete, missing, partial, divergent, and unavailable all-pool `xl.meta` ownership proof, proving only the complete exact tuple permits record cleanup;
- an old reader retaining v2, a mixed fleet blocking v2 writer and migration, and downgrade refusing until v2/control records are drained or exported;
- operator abandon across a crash after one per-copy delete, a lost delete response, a lost progress CAS, a replacement ETag, incomplete topology, and active decommission, proving progress is monotonic, replacements survive, unsafe cases retain evidence, and every case issues zero backend PUT, GET, probe, or DELETE calls;
- canonical export replay, a crash before candidate installation, a lost create response, concurrent admission at the remaining-byte boundary, and actor/cluster count, byte, rate, and concurrency exhaustion, proving duplicate IDs consume no new quota, projected totals never oversubscribe, and admission failure mutates no source evidence.
- **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof.
## `xl.meta` free-version boundary
@@ -472,89 +327,9 @@ Recursive prefix/delete-all is the exception because physical directory removal
- Every code path that removes or overwrites transitioned metadata either atomically leaves an exact free-version owner or enters an already-authorized v6 dispatch.
- If both a journal and free-version are observable, recovery preserves the remote object until version-specific authority and all-pool absence prove a single owner.
- Missing `transitioned-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID.
- Missing `transition-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID.
The following single-record protocol approves how historical objects without RustFS `transitioned-version-state` can be upgraded. It does not approve a bulk scanner or allow destructive cleanup to consume an unproven record.
## Legacy transitioned-version-state reconciliation
### Current
An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. The current GET and free-version cleanup paths reject that state rather than interpreting an empty remote version as unversioned. There is no admin route that repairs this field in `xl.meta`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation.
An explicitly persisted `unknown`, a malformed state, conflicting RustFS/MinIO compatibility keys, an invalid or nil version identifier, and a partial transition tuple are not legacy absence. They remain invalid or ambiguous and fail closed.
### Approved single-record control surface
The approved target is one synchronous, exact logical-version operation. It does not list a bucket or prefix and does not create a durable job:
```text
GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
```
`versionId` is required; the literal `null` is the explicit selector for a locally unversioned object, while an omitted or empty selector is invalid. GET requires `admin:ListTier`. It performs an authoritative all-pool metadata read and a bounded server-side live backend probe, but it never writes metadata or mutates the remote tier. Its response includes the canonical immutable source tuple, every original per-set missing-state representation, the proposed target tuple when one is provable, an opaque reconciliation digest over all three, fleet/topology and tier-generation readiness, and whether POST is ready to attempt migration. GET never labels an unmodified legacy record `migrated`.
POST requires `admin:SetTier`. Its body contains `confirm: true`, the complete source and target tuples, every original per-set representation, and their reconciliation digest returned by a recent GET. The server does not trust a client-supplied state or external assertion: it rereads the object, requires every immutable source field to match exactly after canonicalization, repeats the bounded live probe, and derives the target state itself. For the mutable state/version/destination fields, each set must equal either its original missing-state representation bound by the digest or the exact newly proven target; this is the only accepted partial-retry shape. A missing confirmation, any other stale tuple or digest, or a widened selector is rejected before any write.
The expected tuple binds all evidence whose change could redirect the repair:
- bucket name and incarnation; exact object name, local version ID, data directory, modification time, size, and ETag;
- transition completion status, tier name, canonical remote object name, raw remote-version key presence/value, and raw state-key presence/value under both compatibility prefixes;
- tier-config generation and the `transition-tier-destination-id` binding, including backend type, endpoint/bucket/prefix identity, and the credential-independent backend fingerprint;
- topology generation and the generation/digest of every authoritative `xl.meta` copy found across pools and sets.
The only allowed state derivations are:
| Live proof | Persisted state | Persisted remote version | Remote request meaning |
|---|---|---|---|
| Provider proves versioning disabled and the candidate is present | `KnownDisabled` | Absent/empty | Send no `versionId` |
| Provider proves suspended-version null semantics and the exact candidate is present | `SuspendedNull` | Literal `null` | Send the provider's null-version form |
| Provider proves one exact, nonempty, non-`null` opaque version | `Exact` | That exact opaque identifier | Send that exact `versionId` |
Missing, multiple, changing, or unsupported probe results do not select a state. In particular, a preexisting empty remote-version field is not evidence for `KnownDisabled`, and a client may not nominate `Exact` or `SuspendedNull`.
The POST may write only the derived `transitioned-version-state`, its corresponding `transitioned-versionID` value when the proven model requires one, and the exact `transition-tier-destination-id` binding under both compatibility prefixes in the matching `xl.meta` version. It does not issue remote GET beyond the proof probe, remote PUT, remote DELETE, local object DELETE, free-version cleanup, transaction/journal cleanup, tier-config mutation, restore, or source-payload rewrite. Reconciliation establishes metadata meaning; a later ordinary owner may perform cleanup under its own destructive protocol.
### Outcome contract
POST returns exactly one of the following outcomes and whether it changed bytes. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged:
| Outcome | Meaning and permitted effect |
|---|---|
| `migrated` | All authoritative copies already contain, or were monotonically advanced to, the same proven state and destination identity. Only this outcome makes the record eligible for later ordinary read/delete semantics. |
| `retained-ambiguous` | The tuple is structurally legacy-compatible, but the live probe is missing, multiple, changing, unsupported, or otherwise cannot prove exactly one state. No metadata or remote object is changed. |
| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established, and nothing is changed. |
| `backend-unavailable` | The bound tier generation/destination cannot be acquired, the bounded probe fails, or a metadata quorum/strong readback needed to complete the operation is unavailable. Any already-persisted monotonic subset is retained for an idempotent retry; it is never rolled back. |
HTTP failure detail may distinguish a stale expected tuple, lost fence, timeout, or unavailable quorum, but it must preserve one of these machine-readable outcomes. Logs and audit events include request identity, object identity, tier, generations, outcome, and whether bytes changed; they never include credentials or raw credential-derived configuration.
### Fence, write, and retry order
The approved POST executes the following order. A step that cannot be proven stops the operation without remote mutation:
1. Authenticate `admin:SetTier`, validate the exact single-record selector, `confirm: true`, expected tuple, and digest.
2. Prove every required node advertises the reconciliation format and destination-identity capability; capture the fleet and topology generation. Unknown or unsupported nodes block the writer.
3. Acquire the bucket-lifecycle WRITE fence and validate the bucket incarnation.
4. Acquire the exact tier-config generation lease bound to the expected destination identity.
5. Acquire exact object-version WRITE locks for every owning physical pool/set in stable pool/set order.
6. Perform an authoritative all-pool read, reject duplicate/conflicting ownership, validate every compatibility key, and require each set to match the immutable source tuple plus either its digest-bound original missing-state representation or the exact proposed target.
7. Run one bounded, cancellation-aware live probe through the leased backend. No client or cached probe result is authority.
8. Before writing, revalidate the fleet/topology generation, bucket incarnation and lifecycle fence, tier lease/destination identity, physical owner set, and complete metadata tuple.
9. Write the same derived state and destination identity to each authoritative set with that set's metadata quorum and conditional generation. A timeout or response loss is resolved only by a strong read of that exact set.
10. Strongly reread every authoritative set and revalidate the full tuple, state, destination identity, and topology before returning `migrated`. Release locks and leases in reverse order.
Cross-pool and cross-set partial success is monotonic. The only legal repair edge is `missing state -> one proven {state, remote version, destination identity}`. A retry may accept an already-written subset only when every known copy equals the newly proven target, every remaining copy equals its original missing-state representation captured by the reconciliation digest, and all immutable source fields still match; it then fills only the missing copies. This exact target-plus-original subset is neither stale nor corrupt. The retry never clears a known state, rewrites it to another state, changes destination identity, or rolls a successful set back to missing/`Unknown`. Any other divergent value produces `corrupt`; an unavailable set/readback produces `backend-unavailable`, and destructive cleanup remains blocked until a later strong all-pool read proves complete convergence.
GET takes the same fleet/topology snapshot and authoritative all-pool read but no write locks that imply mutation authority. Because GET is advisory, POST always repeats every fence, read, and live proof rather than promoting the GET result.
### Mixed-version and future batch work
The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. Downgrade is blocked while reconciled records could be rewritten by readers that erase or misinterpret those fields. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient.
Explicit `Unknown`, corruption, and ambiguity remain fail closed for reads that cannot prove non-destructive semantics and for every destructive path. A migrated record becomes ordinary explicit metadata, but reconciliation itself never transfers remote DELETE ownership.
A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a separate durable job identity, create-only admission, lease/CAS checkpoint, bounded pages, per-record expected tuples and outcomes, cancellation/restart semantics, retention, fleet rollout negotiation, and status counters. Implementations must not approximate that protocol by adding a list selector or background loop to the synchronous route.
How historical objects without RustFS transition-version-state can be upgraded safely is an **open design**. It requires a read/repair rule with rollback behavior before destructive cleanup may consume those records.
## Durable namespace receipts during decommission
@@ -562,7 +337,7 @@ A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a
Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current exact ETag conditional-delete primitive. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target.
@@ -572,7 +347,7 @@ Receipts have no enum state. Their legal evolution is `absent -> checkpoint -> m
| Observed state | Unique current owner | Current recovery decision | Destructive admission |
|---|---|---|---|
| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. Eligible v6 journal/manifest and transition-transaction records are removed only by their exact observed ETag | Receipt state grants no remote-delete authority |
| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. An eligible v6 journal/manifest record is directly removed by exact ETag; transition-transaction cleanup follows its documented current unconditional path | Receipt state grants no remote-delete authority |
| Source and target exact identity/checkpoint agree | Decommission coordinator for the run token | Create or CAS-advance the run-scoped receipt | Successor must be monotonic and topology-bound where required |
| Receipt already covers the same successor | Decommission coordinator for the run token | Treat as idempotent | Exact identity/checkpoint only |
| Conflicting receipt, checksum/schema/path error, divergent target record, missing ETag, or non-successor checkpoint | No decommission actor acquires cleanup authority | Fail decommission and retain source | Never overwrite or guess |
@@ -596,7 +371,7 @@ Receipt and expected-manifest create/CAS conflicts retry at most three times. Ex
### Approved target failure matrix
The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. Transition transaction v1 now has create-only installation, exact ETag successor CAS, and conditional terminal deletion, but it still lacks mandatory lost-response exact-successor readback and a renewable durable recovery lease. The manual job's initial write remains non-create-only.
The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. In particular, transition-transaction initial and successor writes and deletes are currently unconditional, while the transition transaction's initial write and the manual job's initial write have neither create-only installation nor mandatory lost-response strong-readback convergence.
| Event | Approved result |
|---|---|
@@ -607,20 +382,19 @@ The matrix below is the normative approved target, not a blanket description of
| Crash after remote DELETE but before journal/free-version cleanup | Retry the same exact idempotent DELETE under the same fences, then conditionally clean local evidence |
| Cancellation | Stop issuing new work, persist monotonic cancellation where the protocol has it, and leave ambiguous durable records for recovery. Cancellation is never rollback proof after authorization |
| Rolling upgrade | Gate writers on the minimum capability required by the format. Known older journal/RPC versions follow their explicit compatibility rule; unknown formats are retained |
| Downgrade | Drain v6 journals and any enabled transition-v2/control protocol before removing their capable workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
| Downgrade | Drain v6 journals before removing all v6-aware workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
| Corrupt or unknown input | Record a diagnosable failure, retain bytes, and block destructive action/completion |
Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have an implemented persisted-format negotiation for rolling downgrade. The approved transition-v2/control gate above is not current behavior. Until the applicable gate is implemented, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have a complete persisted-format negotiation for rolling downgrade. Until one is designed, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
### Current format compatibility decisions
| Family/version | Current reader and writer behavior | Upgrade, downgrade, and ignore rule |
|---|---|---|
| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. V1 remains the only writer format until the approved v2 fleet gate is implemented; a v2 reader never rewrites an active v1 record |
| Transition transaction v2 and recovery-control/export/disposition v1 | Approved target only; no current reader or writer emits these formats | Roll out read support before the homogeneous writer gate; old readers reject and retain. Disable creation and prove all active records drained before downgrade; never rewrite v2 to v1 |
| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. A future schema needs a fleet writer gate and an old-reader retention test before rollout; downgrade behavior is open |
| Tier mutation intent v1; peer RPC v3/v4 | Durable readers/writers require intent v1. New peers accept signed/canonical v3 and v4 RPC; old v3 peers return an exact authenticated unsupported response to v4 | Pause and drain edit/remove/clear across the mixed interval; do not automatically retry v4 as v3. Unknown durable intent is retained and blocks recovery |
| Manual job/scope/task/result v1 | Writers emit the v1 family. Manual-job runtime recovery accepts an uppercase UUID path when both shard strings match its uppercase prefix, then loads the lowercase canonical job by UUID; the decommission validator recomputes the canonical path and rejects that alias. Other decoder/path/checksum failures stop reconciliation. Runtime capabilities advertise `enqueue_only` and `async`, but the async run handler does not consult a fleet capability gate and a direct request creates a job | Runtime recovery still needs exact lowercase canonical-path validation to prevent alias-driven duplicate work. Caller/operator orchestration must verify every required node and fail closed when capability is unknown or unsupported. An automatic server-side fleet gate and persisted downgrade negotiation remain open; unknown records are never ignored as completed work |
| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Never translate empty version ID to known-disabled or authorize remote DELETE. Retain unless the approved exact inspect/export/abandon protocol conditionally removes only the local journal generation |
| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled |
| Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof |
| Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization |
| Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation |
@@ -630,10 +404,10 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr
| Protocol | Current operator/telemetry surface | Current retention | Required follow-up |
|---|---|---|---|
| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Implement the approved v2 lease/takeover, recovery-control status, bounded retry, and backlog metrics |
| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Backlog age/count/state metrics, bounded policy, and durable takeover status |
| Tier mutation intent | Admin mutation response plus recovery diagnostics; no dedicated reconcile API | Peer aborted tombstone through expiry plus skew; ambiguous coordinator/peer records retained | Status/reconcile view for mutation, peer convergence, config generation, and blocked tiers |
| Manual job | POST run response, GET status, DELETE cancel; runtime capabilities advertise both modes | Job/task/result history is indefinite. Terminalizers only best-effort delete the exact scope; startup skips a terminal job with a leftover scope, which remains until a later admission claimant lazily replaces it | Age/count/bytes limit and a terminal-history/scope GC protocol that preserves recovery evidence |
| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Implement the approved single-record inspect/export/disposition, bounded retry controls, and logical backlog metrics |
| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Safe operator inspection/disposition, backlog age/count by version/state, bounded recovery without evidence loss |
| Decommission receipt | Decommission state/events including `receipt_cleanup_failed` | Completion triggers only best-effort receipt/manifest cleanup. Delete failures reported as `receipt_cleanup_failed`, as well as abandoned runs, can leave run-scoped records behind | Run-scoped retention and resume-safe cleanup policy |
Retention is a protocol transition, not raw deletion. Any collector must name its unique owner, minimum age/count/bytes bound, exact terminal or quarantine predicate, readback behavior, decommission interaction, and audit/metric output. It may not collect a record solely because it is old.
+6 -89
View File
@@ -39,11 +39,11 @@ Internal metadata is stored under both `x-rustfs-internal-<suffix>` and `x-minio
|--------|---------|
| `transition-status` | `"complete"` when tiered |
| `transitioned-object` | tier key path (stored without the tier prefix; `get_dest` adds it) |
| `transitioned-versionID` | Provider version identifier: current exact UTF-8 text, legacy RustFS raw UUID bytes, MinIO's empty unversioned value, or absent for some historical unversioned records. Interpret it only with `transitioned-version-state` or a live compatibility probe. |
| `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) |
| `transition-tier` | tier name |
| `tier-free-versionID` | delete-marker version for free-version sweep |
Legacy raw UUID values must reject empty, malformed, and nil UUIDs (regression covered in `crates/filemeta/src/filemeta/version.rs` tests):
Reading binary values must reject empty, malformed, and nil values (regression covered in `crates/filemeta/src/filemeta/version.rs` tests):
```rust
get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
@@ -52,7 +52,7 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
// None for: absent key, wrong-length bytes, nil UUID
```
`transition_version_id == None` means only that no usable legacy UUID projection exists; it does not prove the remote bucket's versioning model. Only an explicit `KnownDisabled` state authorizes ordinary GET/DELETE to omit `versionId`. A missing state with an absent or empty version key remains `Unknown` and requires the bounded compatibility probe or the approved reconcile workflow; it never directly authorizes cleanup. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure.
`transition_version_id == None` means the tier bucket is unversioned; the GET/DELETE against the tier must then send no `versionId` parameter. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure.
## Inspect xl.meta directly
@@ -64,8 +64,8 @@ cargo build -p rustfs-filemeta --example dump_fileinfo
| Output | Meaning |
|---|---|
| `transition_ver_id: <none>` | No usable legacy UUID projection exists. Inspect `transitioned-version-state` and the raw compatibility keys; do not infer unversioned semantics. |
| `transition_ver_id: <uuid>` | A legacy UUID representation decoded successfully. It is not destructive authority unless the persisted state or reconcile proof establishes the exact remote model. |
| `transition_ver_id: <none>` | No versionId will be sent to the tier (correct for a non-versioned tier bucket). |
| `transition_ver_id: <uuid>` | That UUID will be sent as `?versionId=<uuid>`. |
There is one `xl.meta` per erasure shard disk (`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should be identical. `dump_versions` (same crate) lists every version in a file.
@@ -80,7 +80,7 @@ RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs ...
| `fetching transitioned object from tier` | DEBUG | Emitted before the tier request. |
| `tier GET failed` | ERROR | Includes `tier_version_id`. |
If the version keys are empty while `transitioned-version-state` is absent, the record has the historical MinIO unversioned shape but still remains `Unknown`; only the compatibility probe or reconcile protocol may prove `KnownDisabled`. If state is explicitly `KnownDisabled`, no `versionId` is sent.
If both `x-rustfs-internal-transitioned-versionID` and `x-minio-internal-transitioned-versionID` are the empty string, the object was transitioned to a non-versioned tier bucket and no versionId must be sent.
## Manual transition run
@@ -154,89 +154,6 @@ Historical transition transactions in `upload_outcome_unknown` state can use an
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
## Inspect and disposition retained recovery records
This section describes an **approved target that is not implemented yet**. Current servers do not expose the routes below and continue to quarantine tier-delete journal v1/v2 records. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees.
The approved read-only inventory is bounded and paginated:
```text
GET /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
GET /rustfs/admin/v3/ilm/recovery/records/<control-id>
```
List and redacted inspect require `admin:ListTier`. The server reconstructs the canonical source identity, strongly reads every authoritative copy, and reports one logical record with its schema, classification (`retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`), stable reason code, copy/content digests, retry deadline/counters, fleet readiness, scan completeness, and decommission coverage. It does not return raw legacy bytes, object/version names, endpoints, credentials, or provider error text in the default JSON. Incomplete pool coverage, divergent copies, a missing ETag, corruption, or a truncated page without a continuation marker is fail-closed and cannot produce an actionable receipt.
Inspect returns a 15-minute opaque observation receipt. It binds the authenticated actor, canonical record, every source copy/ETag/digest, topology/fleet generation, requested action class, issue/expiry time, and nonce. The receipt prevents a stale request from widening its target; it is not cleanup authority.
For a strictly decoded v1/v2 tier-delete journal, the approved evidence-preserving flow is:
1. Inspect the exact record and independently decide whether retaining the local cleanup obligation is still useful.
2. With `admin:SetTier`, create an immutable server-side export from the current observation receipt. The export contains the exact raw journal bytes and copy manifest, is installed create-only at the canonical digest-derived export ID, strongly read back, and downloaded through a no-store attachment response:
```text
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
{ "action": "export", "observation_receipt": "<opaque>" }
GET /rustfs/admin/v3/ilm/recovery/exports/<export-id>
```
3. Only after preserving that export, submit a fresh exact disposition with `admin:SetTier`:
```json
POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
{
"action": "abandon_remote_cleanup",
"confirm": true,
"acknowledge_remote_cleanup_abandoned": true,
"observation_receipt": "<opaque>",
"export_id": "<export-id>",
"export_sha256": "<sha256>",
"reason_code": "<bounded-operator-reason>"
}
```
The last action removes only the exact local v1/v2 journal generations by per-copy `If-Match` after a durable `Prepared` disposition receipt and fresh all-member capability proof. The receipt advances `Prepared -> Applying -> Completed` and records a monotonic per-copy `confirmed_absent` set. If the server deletes copy A and crashes before recording progress, recovery may confirm A absent under the unchanged source/control, topology, process-epoch, migration, and decommission proofs, persist that progress, and continue with still-exact copy B. A replacement ETag is always a conflict; recovery never widens the immutable copy manifest.
The action never creates a tier client, probes a backend, or issues remote PUT/GET/DELETE. Its meaning is deliberately narrow: the operator accepts that remote storage may leak and abandons RustFS cleanup after preserving evidence. A changed copy, active decommission, missing member, topology/process restart, incomplete read, or uncertain replacement proof retains the evidence. Success requires every bound copy be durably confirmed absent, a fresh all-member/decommission proof, and the disposition receipt durably `Completed`; response loss resumes only the same canonical operation ID.
Canonical replay of an identical export/disposition consumes no new quota. New operations require a complete artifact inventory and are refused before source mutation when the projected retained total, including the fully encoded candidate, would exceed 10,000 exports, 10,000 disposition receipts, 1 GiB of encoded export data, or 256 MiB of encoded control/disposition data. The quota decision, create-only installation, and exact readback share one cluster-scoped admission WRITE lock. That lock is always acquired before control/source/disposition and physical metadata locks and is released before disposition `Applying` or any source deletion; callers never acquire it while holding those inner guards. A crash before installation consumes no capacity, and lost installation response is resolved by canonical readback under the same serialized order, so concurrent nodes cannot oversubscribe a stale snapshot. Admission is also limited to ten new creations per actor per minute, 100 cluster-wide per minute, 32 concurrent exports, and eight concurrent dispositions. Capacity pressure never evicts recovery evidence or blocks ordinary object I/O; the collector examines at most 100 terminal artifacts per minute.
Malformed/unsupported records and journal v3-v6 cannot use abandon. Known-version and v6 manifest ownership must converge through their normal exact recovery protocol. Operators may inspect, export, and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or version semantics.
Automatic retry state survives restart. Retryable transport/quorum failures use a 60-second exponential base capped at one hour and a deterministic 80-to-100-percent multiplier, so jitter never increases the capped delay. After 32 consecutive failures or seven days from the first persisted failure, automatic work stops at `operator_required`. Unsupported or ambiguous evidence goes directly to `retained_ambiguous`/`operator_required`; age alone never deletes it. Resolved controls, immutable exports, and completed disposition receipts have minimum 30-day, 90-day, and 365-day retention respectively, and are collected only after exact source absence, decommission, successor, and audit checks.
The full schema, lease, mixed-version, retry, privacy, and metric requirements are in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#bounded-recovery-control-and-operator-disposition).
## Reconcile legacy transition-version metadata
This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version.
The approved interface is synchronous and accepts exactly one bucket/object/local-version tuple:
```text
GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
```
`versionId` is required; use the literal `null` for a locally unversioned object. An omitted or empty selector is invalid. GET requires `admin:ListTier`. It reports the authoritative all-pool tuple, destination identity, fleet/topology readiness, live probe classification, opaque expected-tuple digest, and a machine-readable diagnosis. It returns `ready-to-migrate`, not `migrated`, when a missing state is provable because GET is read-only.
POST requires `admin:SetTier`, `confirm: true`, and the complete immutable source tuple, original per-set missing-state representations, proposed target, and reconciliation digest returned by GET. The server rereads every authoritative copy and repeats the bounded live backend probe; provider console output or an operator-supplied state is diagnostic evidence only, never write authority. A retry accepts only copies that still match their digest-bound original representation or already equal the exact proven target; any other divergence is stale or corrupt. The server may persist only one of these exact state/version pairs, together with the bound destination identity:
| Proven remote model | State | Version value |
|---|---|---|
| Versioning disabled | `KnownDisabled` | Empty/absent; later requests omit `versionId` |
| Versioning suspended null object | `SuspendedNull` | Literal `null` |
| One exact version | `Exact` | Exact nonempty, non-`null` opaque identifier |
The response outcome is `migrated`, `retained-ambiguous`, `corrupt`, or `backend-unavailable`. `migrated` means strong all-pool readback proved the same state and destination identity on every authoritative copy; it can be idempotent with `changed=false`. Ambiguous/missing/multiple probe results are retained, and explicit `Unknown`, malformed or conflicting dual keys, nil identifiers, partial tuples, or copies outside the exact `{original missing representation, proven target}` retry subset fail closed. An unavailable backend, tier generation, metadata quorum, or required strong readback reports `backend-unavailable`; a monotonic partial write is retained for retry and never rolled back.
The POST does not issue remote DELETE or PUT, remove local data, create a free-version, clean a transaction/journal, or change tier configuration. It holds the approved fleet/topology, bucket-lifecycle, exact tier-generation/destination, and stable all-pool object-version fences across authoritative reread and the bounded probe; it rechecks them before quorum writes and after strong readback. A fleet containing a node that cannot preserve the explicit state/destination binding is inspect-only, and a cross-pool first match is never enough.
There is intentionally no bucket, prefix, or fleet selector. Batch repair requires a separate durable, resumable job protocol and remains future work. Until the single-record route is implemented, retain affected metadata, use external inspection only for diagnosis, and never hand-edit `xl.meta` or enable remote cleanup by assuming that an empty version field means an unversioned tier.
The full approved fence, quorum, cross-set retry, destination-binding, and mixed-version contract is specified in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#legacy-transitioned-version-state-reconciliation).
## Invariant: local-first expiry ordering
`expire_transitioned_object` deletes local metadata first (making the object unreachable) and leaves a persisted free-version for remote-tier cleanup. `tier_free_version_recovery.rs` scans and re-enqueues that record; the lifecycle worker's `cleanup_free_version_exact` in `bucket_lifecycle_ops.rs` performs the fenced remote delete, local-marker cleanup, and rescan. Never remove a remote tier version while live local metadata still points at it: doing so lets a concurrent GET read a stored version_id whose remote version is already gone and fail with `NoSuchVersion`.
+69 -121
View File
@@ -80,82 +80,6 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result<String> {
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
}
fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> {
match config.tier_type {
TierType::S3 => {
let _ = config
.s3
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?;
}
TierType::Wasabi => config.name = wasabi_payload_name(config)?,
TierType::RustFS => {
config.name = config
.rustfs
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
.name
.clone();
}
TierType::MinIO => {
config.name = config
.minio
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
.name
.clone();
}
TierType::Aliyun => {
config.name = config
.aliyun
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
.name
.clone();
}
TierType::Tencent => {
config.name = config
.tencent
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
.name
.clone();
}
TierType::Huaweicloud => {
config.name = config
.huaweicloud
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
.name
.clone();
}
TierType::Azure => {
config.name = config
.azure
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
.name
.clone();
}
TierType::GCS => {
let _ = config
.gcs
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?;
}
TierType::R2 => {
config.name = config
.r2
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
.name
.clone();
}
TierType::Unsupported => {}
}
Ok(())
}
fn spawn_transition_tier_config_propagation(action: &'static str) {
if let Some(notification_sys) = current_notification_system() {
debug!(
@@ -339,7 +263,75 @@ impl Operation for AddTier {
let mut args: TierConfig = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
normalize_add_tier_payload_name(&mut args)?;
match args.tier_type {
TierType::S3 => {
args.name = args
.s3
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
.name;
}
TierType::Wasabi => {
args.name = wasabi_payload_name(&args)?;
}
TierType::RustFS => {
args.name = args
.rustfs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
.name;
}
TierType::MinIO => {
args.name = args
.minio
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
.name;
}
TierType::Aliyun => {
args.name = args
.aliyun
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
.name;
}
TierType::Tencent => {
args.name = args
.tencent
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
.name;
}
TierType::Huaweicloud => {
args.name = args
.huaweicloud
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
.name;
}
TierType::Azure => {
args.name = args
.azure
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
.name;
}
TierType::GCS => {
args.name = args
.gcs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?
.name;
}
TierType::R2 => {
args.name = args
.r2
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
.name;
}
_ => (),
}
debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
@@ -1158,44 +1150,6 @@ mod tests {
assert_eq!(err.message(), Some("missing Wasabi configuration"));
}
#[test]
fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() {
for (provider, wire) in [
(
"S3",
serde_json::json!({
"Type": "s3",
"Name": "COLD-S3",
"S3": {
"Endpoint": "https://s3.example.invalid",
"AccessKey": "access",
"SecretKey": "secret",
"Bucket": "archive"
}
}),
),
(
"GCS",
serde_json::json!({
"Type": "gcs",
"Name": "COLD-GCS",
"GCS": {
"Endpoint": "https://storage.googleapis.com",
"Creds": "e30=",
"Bucket": "archive"
}
}),
),
] {
let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode");
let expected = config.name.clone();
normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary");
assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared");
}
}
#[test]
fn resolve_tier_name_prefers_path_parameter() {
let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
@@ -1802,11 +1756,5 @@ mod tests {
}
assert!(!production.contains("check_key_valid(get_session_token"));
let add_tier = source_block(production, "impl Operation for AddTier");
assert!(
add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"),
"AddTier must preserve canonical top-level provider names through the tested boundary helper"
);
}
}
+5 -5
View File
@@ -170,12 +170,12 @@ use s3s::dto::{
DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts,
GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold,
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput,
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus,
SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput,
ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader,
Timestamp, TimestampFormat, WebsiteRedirectLocation,
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm,
SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption,
ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat,
WebsiteRedirectLocation,
};
use s3s::header::X_AMZ_RESTORE;
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
+57 -166
View File
@@ -165,49 +165,18 @@ impl DefaultObjectUsecase {
validate_table_catalog_object_mutation(&bucket, &object).await?;
// Typed S3 errors on every RestoreObject failure (backlog#2205): a
// `Custom` code serializes as a generic 500, which makes SDK clients
// retry client errors and conflicts alike.
let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
// SELECT-type restore is not supported (backlog#1341). The restore
// path can only write the retrieved bytes back to the source key, so
// honouring a SELECT request overwrote the source object with
// SELECT-only metadata (dropping `x-amz-restore`, user metadata and
// tags on an unversioned bucket, or publishing a bogus latest version
// on a versioned one) while never writing anything to
// `OutputLocation.S3`. Reject before any guard, metadata write or
// fabricated `x-amz-restore-output-path` response header.
if rreq
.type_
.as_ref()
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
{
return Err(S3Error::with_message(
S3ErrorCode::NotImplemented,
"SELECT restore requests are not supported.",
));
}
let rreq = rreq.ok_or_else(|| {
S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required")
})?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// Validate the request shape before taking any lock or reading the
// object: a malformed request or an illegal `Days` value is a client
// error, and the validator messages are static — they carry no
// backend or credential detail.
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("Restore object validation failed: {e}"),
));
}
let version_id_str = version_id.clone().unwrap_or_default();
let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
.await
.map_err(ApiError::from)?;
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?;
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
// `apply_bucket_generation_guard` deliberately tolerates a missing guard
// (only the S3 access layer installs one), so this must not hard-require
@@ -223,7 +192,11 @@ impl DefaultObjectUsecase {
}
};
let restore_operation_id = Some(Uuid::new_v4());
// SELECT-type restores skip both the ongoing check and the metadata
// write below, so the accept guard would protect nothing for them —
// they keep the plain (read-locked) accept path.
let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT");
let restore_operation_id = (!is_select).then(Uuid::new_v4);
let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id {
Some(
store
@@ -237,10 +210,10 @@ impl DefaultObjectUsecase {
// Hold the restore-accept guard across the restore-status read, the
// ongoing/already-restored decision, and the metadata write below, so
// two concurrent POST ?restore cannot both observe ongoing=false and
// both start a copy-back (backlog#1304). Reads and writes inside this
// scope run with no_lock; the guard is dropped before the copy-back is
// spawned so it never blocks readers.
// two concurrent (non-SELECT) POST ?restore cannot both observe
// ongoing=false and both start a copy-back (backlog#1304). Reads and
// writes inside this scope run with no_lock; the guard is dropped
// before the copy-back is spawned so it never blocks readers.
// Contention on the accept guard (e.g. a concurrent accept or an
// in-flight commit on the same object) is transient — answer 503
// SlowDown so SDK clients back off and retry instead of treating it
@@ -249,7 +222,9 @@ impl DefaultObjectUsecase {
if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id {
return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into());
}
let mut accept_guard = {
let mut accept_guard = if is_select {
None
} else {
let guard = store
.acquire_restore_accept_guard(&bucket, &object)
.await
@@ -258,17 +233,24 @@ impl DefaultObjectUsecase {
Some(guard)
};
// A missing key or version must stay NoSuchKey / NoSuchVersion, and an
// authorization or storage failure must keep its own identity, so map
// the storage error instead of flattening it (backlog#2205).
let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
let mut obj_info = store
.get_object_info(&bucket, &object, &opts)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?;
// Restoring an object that was never transitioned is the S3
// InvalidObjectState case, not an internal error.
// Check if object is in a transitioned state
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return Err(S3Error::with_message(
S3ErrorCode::InvalidObjectState,
"The operation is not valid for the object's storage class.",
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
"restore object failed.",
));
}
// Validate restore request
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrValidRestoreObject".into()),
format!("Restore object validation failed: {}", e),
));
}
@@ -278,7 +260,7 @@ impl DefaultObjectUsecase {
// would create an ABBA cycle. If the probe succeeds, reacquire and
// re-read the object before replacing the exact orphan generation.
let mut superseded_worker_guard = None;
if obj_info.restore_ongoing {
if obj_info.restore_ongoing && !is_select {
match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) {
OngoingRestoreRecovery::ActiveOrUnsafe => {
return Err(S3Error::with_message(
@@ -311,11 +293,13 @@ impl DefaultObjectUsecase {
.map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?,
);
opts.no_lock = true;
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| {
S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.")
})?;
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return Err(S3Error::with_message(
S3ErrorCode::InvalidObjectState,
"The operation is not valid for the object's storage class.",
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
"restore object failed.",
));
}
if obj_info.restore_ongoing {
@@ -343,11 +327,11 @@ impl DefaultObjectUsecase {
remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID);
remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK);
let mut header = HeaderMap::new();
let event_object_info = obj_info.clone();
let obj_info_ = obj_info.clone();
// Scopes the accept-guarded metadata write: everything below runs
// inside the accept critical section, which is released right after.
{
if !is_select {
obj_info.metadata_only = true;
metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string());
let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| {
@@ -419,7 +403,7 @@ impl DefaultObjectUsecase {
&restore_dst_opts,
)
.await
.map_err(ApiError::from)?;
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?;
rustfs_scanner::record_dirty_usage_bucket(&bucket);
#[cfg(test)]
maybe_pause_after_restore_status_commit(&bucket, &object).await;
@@ -445,6 +429,17 @@ impl DefaultObjectUsecase {
drop(accept_guard);
drop(restore_bucket_lifecycle_guard);
// Handle output location for SELECT requests
if let Some(output_location) = &rreq.output_location
&& let Some(s3) = &output_location.s3
&& !s3.bucket_name.is_empty()
{
let restore_object = Uuid::new_v4().to_string();
if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() {
header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value);
}
}
// Spawn restoration task in the background. Pin the copy-back to the
// version the accept resolved and flagged: with a versionless request
// on a versioned bucket, a PUT landing between the accept and the
@@ -504,7 +499,7 @@ impl DefaultObjectUsecase {
restore_output_path: None,
};
helper = helper.object(event_object_info).version_id(version_id_str);
let result = Ok(S3Response::new(output));
let result = Ok(S3Response::with_headers(output, header));
let _ = helper.complete(&result);
result
}
@@ -634,28 +629,6 @@ mod tests {
assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe);
}
fn restore_request(days: Option<i32>) -> RestoreRequest {
RestoreRequest {
days,
description: None,
glacier_job_parameters: None,
output_location: None,
select_parameters: None,
tier: None,
type_: None,
}
}
fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput {
RestoreObjectInput::builder()
.bucket(bucket.to_string())
.key(key.to_string())
.restore_request(Some(rreq))
.build()
.expect("restore input should build")
}
/// backlog#2205: a missing restore body is a client error, not a 500.
#[tokio::test]
async fn execute_restore_object_rejects_missing_restore_request() {
let input = RestoreObjectInput::builder()
@@ -668,92 +641,10 @@ mod tests {
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_restore_object(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
}
/// backlog#1341: a SELECT restore must be rejected outright — the restore
/// path can only write back to the source key, never to
/// `OutputLocation.S3`. Rejection happens before the store is resolved, so
/// an uninitialized usecase still answers NotImplemented rather than the
/// InternalError every request that gets past this point returns.
#[tokio::test]
async fn execute_restore_object_rejects_select_type() {
let mut rreq = restore_request(None);
rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT));
let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST);
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_restore_object(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
/// backlog#2205: every RestoreObject failure that reaches storage must
/// keep its typed S3 identity. Before this, a missing key, a malformed
/// version-id, an illegal `Days` and an object that was never transitioned
/// all collapsed into `Custom(...)` codes, which serialize as a retryable
/// HTTP 500.
#[tokio::test]
#[serial_test::serial]
async fn execute_restore_object_maps_failures_to_typed_s3_errors() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create restore test bucket");
let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec());
store
.put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default())
.await
.expect("put untransitioned test object");
let usecase = DefaultObjectUsecase::with_context(Some(context));
// An illegal `Days` is a client error, rejected before any lock or
// object read.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "local-object", restore_request(Some(0))),
Method::POST,
))
.await
.expect_err("days=0 must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
// A malformed version-id keeps InvalidArgument instead of being
// flattened inside `post_restore_opts`.
let mut input = restore_input(&bucket, "local-object", restore_request(Some(1)));
input.version_id = Some("not-a-uuid".to_string());
let err = usecase
.execute_restore_object(build_request(input, Method::POST))
.await
.expect_err("malformed version-id must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
// A missing key stays NoSuchKey.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "missing-object", restore_request(Some(1))),
Method::POST,
))
.await
.expect_err("missing key must be rejected");
assert_eq!(err.code(), &S3ErrorCode::NoSuchKey);
// Restoring an object that was never transitioned is the S3
// InvalidObjectState case, not an internal error.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "local-object", restore_request(Some(1))),
Method::POST,
))
.await
.expect_err("untransitioned object must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState);
match err.code() {
S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"),
code => panic!("unexpected error code: {:?}", code),
}
}
#[tokio::test]
+1 -2
View File
@@ -368,8 +368,7 @@ pub(crate) mod bucket {
version_id: &str,
bucket: &str,
object: &str,
) -> Result<crate::storage::storage_api::StorageObjectOptions, crate::storage::storage_api::StorageError>
{
) -> Result<crate::storage::storage_api::StorageObjectOptions, std::io::Error> {
crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(
version_id, bucket, object,
)
+2 -6
View File
@@ -14,8 +14,6 @@
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
use std::alloc::{GlobalAlloc, Layout};
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
use std::ptr::NonNull;
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
#[derive(Default)]
@@ -37,10 +35,8 @@ unsafe impl GlobalAlloc for MiMallocAllocator {
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
// SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's
// dealloc contract, and layout.size() is the original allocation size.
let ptr = unsafe { NonNull::new_unchecked(ptr) };
unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) }
// SAFETY: ptr came from this allocator and layout.size() is the original allocation size.
unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+162 -6
View File
@@ -14,6 +14,8 @@
use const_str::concat;
use shadow_rs::shadow;
use std::path::Path;
use std::process::Command;
shadow!(build);
@@ -45,6 +47,10 @@ pub const DISPLAY_VERSION: &str = {
type VersionParseResult = Result<(u32, u32, u32, Option<String>), Box<dyn std::error::Error>>;
fn build_version_override() -> Option<&'static str> {
BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty())
}
fn version_ref(version: &str) -> String {
if version.starts_with("refs/tags/") || version.starts_with('@') {
version.to_string()
@@ -55,7 +61,91 @@ fn version_ref(version: &str) -> String {
#[allow(clippy::const_is_empty)]
pub fn get_version() -> String {
version_ref(DISPLAY_VERSION)
if let Some(version) = build_version_override() {
return version_ref(version);
}
// Get the latest tag
if let Ok(latest_tag) = get_latest_tag() {
// Check if current commit is newer than the latest tag
if is_head_newer_than_tag(&latest_tag) {
// If current commit is newer, increment the version number
if let Ok(new_version) = increment_version(&latest_tag) {
return format!("refs/tags/{new_version}");
}
}
// If current commit is the latest tag, or version increment failed, return current tag
return format!("refs/tags/{latest_tag}");
}
// If no tag exists, use original logic
if !build::TAG.is_empty() {
format!("refs/tags/{}", build::TAG)
} else if !build::SHORT_COMMIT.is_empty() {
format!("@{}", build::SHORT_COMMIT)
} else {
format!("refs/tags/{}", build::PKG_VERSION)
}
}
/// Get the latest git tag
fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?;
if output.status.success() {
let tag = String::from_utf8(output.stdout)?;
Ok(tag.trim().to_string())
} else {
Err("Failed to get latest tag".into())
}
}
/// Check if current HEAD is newer than specified tag
fn is_head_newer_than_tag(tag: &str) -> bool {
is_head_newer_than_tag_in(Path::new("."), tag)
}
fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool {
let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output();
let tag_commit = Command::new("git")
.current_dir(repo)
.args(["rev-list", "-n", "1", tag])
.output();
let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else {
return false;
};
if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout {
return false;
}
let output = Command::new("git")
.current_dir(repo)
.args(["merge-base", "--is-ancestor", tag, "HEAD"])
.output();
match output {
Ok(result) => result.status.success(),
Err(_) => false,
}
}
/// Increment version number (increase patch version)
fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>> {
// Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
let (major, minor, patch, pre_release) = parse_version(version)?;
// If there's a pre-release identifier, increment the pre-release version number
if let Some(pre) = pre_release
&& let Some(new_pre) = increment_pre_release(&pre)
{
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
}
// Otherwise increment patch version number
Ok(format!("{major}.{minor}.{}", patch + 1))
}
/// Parse version number
@@ -76,6 +166,28 @@ pub fn parse_version(version: &str) -> VersionParseResult {
Ok((major, minor, patch, pre_release))
}
/// Increment pre-release version number
fn increment_pre_release(pre_release: &str) -> Option<String> {
// Handle pre-release versions like "alpha.19"
let parts: Vec<&str> = pre_release.split('.').collect();
if parts.len() == 2
&& let Ok(num) = parts[1].parse::<u32>()
{
return Some(format!("{}.{}", parts[0], num + 1));
}
// Handle pre-release versions like "alpha19"
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
let prefix = &pre_release[..=pos];
let suffix = &pre_release[pos + 1..];
if let Ok(num) = suffix.parse::<u32>() {
return Some(format!("{prefix}{}", num + 1));
}
}
None
}
/// Clean version string - removes common prefixes
pub fn clean_version(version: &str) -> String {
version
@@ -172,6 +284,34 @@ mod tests {
use super::*;
use tracing::debug;
fn run_git(repo: &Path, args: &[&str]) {
let status = Command::new("git").current_dir(repo).args(args).status().unwrap();
assert!(status.success(), "git command failed: git {}", args.join(" "));
}
#[test]
fn test_is_head_newer_than_tag_requires_strict_descendant() {
let repo = tempfile::tempdir().unwrap();
run_git(repo.path(), &["init", "--quiet"]);
run_git(repo.path(), &["config", "user.name", "RustFS Tests"]);
run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]);
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]);
run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]);
assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3"));
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]);
assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3"));
}
#[test]
fn build_version_override_is_used_for_current_version_when_set() {
if let Some(version) = build_version_override() {
assert_eq!(get_version(), version_ref(version));
}
}
#[test]
fn version_ref_keeps_existing_ref_prefixes() {
assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3");
@@ -179,11 +319,6 @@ mod tests {
assert_eq!(version_ref("@abc123"), "@abc123");
}
#[test]
fn get_version_uses_build_metadata() {
assert_eq!(get_version(), version_ref(DISPLAY_VERSION));
}
#[test]
fn test_parse_version() {
// Test standard version parsing
@@ -201,6 +336,27 @@ mod tests {
assert_eq!(pre_release, Some("alpha.19".to_string()));
}
#[test]
fn test_increment_pre_release() {
// Test alpha.19 -> alpha.20
assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string()));
// Test beta.5 -> beta.6
assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string()));
// Test unparsable case
assert_eq!(increment_pre_release("unknown"), None);
}
#[test]
fn test_increment_version() {
// Test pre-release version increment
assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20");
// Test standard version increment
assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1");
}
#[test]
fn test_version_format() {
// Test if version format starts with refs/tags/