diff --git a/.config/nextest.toml b/.config/nextest.toml index c6eaf1908..bc57b8b8f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.default.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.default.scripts]] @@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ setup = 'ecstore-large-stack' [[profile.ci.scripts]] -filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' +filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' [[profile.ci.scripts]] diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 00486eb96..995cc9b23 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -152,6 +152,60 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + STEPS_TABLE="/tmp/rustfs-heal-steps.md" + python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' + import re + import sys + + log_file, out_file = sys.argv[1], sys.argv[2] + ansi = re.compile(r'\x1b\[[0-9;]*m') + step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$') + ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$') + result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$') + + steps = {} + order = [] + version = None + version_node = None + verdict = None + verdict_detail = '' + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: + for raw in fh: + line = ansi.sub('', raw).strip() + m = step_re.match(line) + if m: + n, desc, status = m.group(1), m.group(2), m.group(3) + if n not in steps: + order.append(n) + steps[n] = (desc, status) # later lines win (fail after pass) + continue + m = ver_re.match(line) + if m: + version, version_node = m.group(1), m.group(2) + continue + m = result_re.match(line) + if m: + verdict, verdict_detail = m.group(1), m.group(2) + except FileNotFoundError: + pass + + with open(out_file, 'w', encoding='utf-8') as out: + out.write('## Step Results\n\n') + if version: + node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else '' + out.write(f'- Version under test: **{version}**{node_note}\n') + if verdict: + out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n') + out.write('\n') + out.write('| Step | Description | Result |\n') + out.write('| --- | --- | --- |\n') + for n in sorted(order, key=int): + desc, status = steps[n] + out.write(f'| {n} | {desc} | {status} |\n') + if not order: + out.write('| - | - | NOT RUN (no step result lines found) |\n') + PY { echo "# RustFS heal test report" echo "" @@ -160,6 +214,8 @@ jobs: echo "- Package: ${PACKAGE_SOURCE}" echo "- Test Step Outcome: ${{ steps.test.outcome }}" echo "" + cat "${STEPS_TABLE}" || true + echo "" echo "## Log tail" echo '```text' tail -n 200 "${LOG_FILE}" || true diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index d7c0d6abc..d7002c446 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -380,6 +380,60 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md" + python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' + import re + import sys + + log_file, out_file = sys.argv[1], sys.argv[2] + ansi = re.compile(r'\x1b\[[0-9;]*m') + step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$') + ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$') + result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$') + + steps = {} + order = [] + version = None + version_node = None + verdict = None + verdict_detail = '' + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: + for raw in fh: + line = ansi.sub('', raw).strip() + m = step_re.match(line) + if m: + n, desc, status = m.group(1), m.group(2), m.group(3) + if n not in steps: + order.append(n) + steps[n] = (desc, status) # later lines win (fail after pass) + continue + m = ver_re.match(line) + if m: + version, version_node = m.group(1), m.group(2) + continue + m = result_re.match(line) + if m: + verdict, verdict_detail = m.group(1), m.group(2) + except FileNotFoundError: + pass + + with open(out_file, 'w', encoding='utf-8') as out: + out.write('## Step Results\n\n') + if version: + node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else '' + out.write(f'- Version under test: **{version}**{node_note}\n') + if verdict: + out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n') + out.write('\n') + out.write('| Step | Description | Result |\n') + out.write('| --- | --- | --- |\n') + for n in sorted(order, key=int): + desc, status = steps[n] + out.write(f'| {n} | {desc} | {status} |\n') + if not order: + out.write('| - | - | NOT RUN (no step result lines found) |\n') + PY { echo "# RustFS pool expansion test report" echo "" @@ -389,6 +443,8 @@ jobs: echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}" echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}" echo "" + cat "${STEPS_TABLE}" || true + echo "" echo "## Log tail" echo '```text' tail -n 200 "${LOG_FILE}" || true diff --git a/Cargo.lock b/Cargo.lock index 2aba26999..e9e7d9fe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,9 +1328,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -1368,9 +1368,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.2" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -5103,9 +5103,9 @@ dependencies = [ [[package]] name = "hickory-net" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1" dependencies = [ "async-trait", "cfg-if", @@ -5127,9 +5127,9 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d" dependencies = [ "data-encoding", "idna", @@ -5147,9 +5147,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f" dependencies = [ "cfg-if", "futures-util", @@ -5226,9 +5226,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737" +checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592" dependencies = [ "arc-swap", "async-channel", @@ -5240,7 +5240,6 @@ dependencies = [ "futures-util", "hdrhistogram", "hotpath-macros", - "hotpath-meta", "http 1.5.0", "libc", "object 0.36.7", @@ -5260,30 +5259,15 @@ dependencies = [ [[package]] name = "hotpath-macros" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1" +checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] -[[package]] -name = "hotpath-macros-meta" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b" - -[[package]] -name = "hotpath-meta" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b" -dependencies = [ - "hotpath-macros-meta", -] - [[package]] name = "htmlescape" version = "0.3.1" @@ -5642,18 +5626,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "internal-russh-num-bigint" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" -dependencies = [ - "num-integer", - "num-traits", - "rand 0.10.2", - "rand_core 0.10.1", -] - [[package]] name = "io-uring" version = "0.7.14" @@ -5904,17 +5876,20 @@ dependencies = [ [[package]] name = "kafka-protocol" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2" +checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda" dependencies = [ "anyhow", "bytes", "crc", "crc32c", + "flate2", "indexmap 2.14.1", - "paste", + "lz4", + "snap", "uuid", + "zstd", ] [[package]] @@ -6923,6 +6898,8 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", + "rand 0.10.2", + "rand_core 0.10.1", ] [[package]] @@ -7515,9 +7492,9 @@ dependencies = [ [[package]] name = "pageant" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791" +checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce" dependencies = [ "base16ct 1.0.0", "byteorder", @@ -7606,12 +7583,6 @@ dependencies = [ "phc", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "path-absolutize" version = "4.0.1" @@ -9250,9 +9221,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.63.1" +version = "0.63.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031" +checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd" dependencies = [ "aes 0.9.3", "aws-lc-rs", @@ -9281,13 +9252,12 @@ dependencies = [ "hex-literal", "hmac 0.13.0", "inout 0.2.2", - "internal-russh-num-bigint", "keccak", "log", "md5", "ml-kem", "module-lattice", - "num-bigint 0.4.8", + "num-bigint 0.5.1", "p256 0.14.0", "p384 0.14.0", "p521", @@ -9978,11 +9948,11 @@ dependencies = [ [[package]] name = "rustfs-kafka" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96" +checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "fnv", "hmac 0.13.0", @@ -9992,22 +9962,22 @@ dependencies = [ "pbkdf2 0.13.0", "rand 0.10.2", "rustls", - "rustls-native-certs", "sha2 0.11.0", "socket2", "thiserror 2.0.20", "tracing", "twox-hash", + "uuid", "webpki-roots 1.0.9", ] [[package]] name = "rustfs-kafka-async" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630" +checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "hmac 0.13.0", "kafka-protocol", @@ -10016,11 +9986,11 @@ dependencies = [ "rand 0.10.2", "rustfs-kafka", "rustls", - "rustls-native-certs", "sha2 0.11.0", "tokio", "tokio-rustls", "tracing", + "uuid", "webpki-roots 1.0.9", ] @@ -10194,18 +10164,18 @@ dependencies = [ [[package]] name = "rustfs-mimalloc" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15" +checksum = "46a7b69356718defa4060de3059e609c9da6619c35924fefef55d902ccd94958" dependencies = [ "rustfs-mimalloc-sys", ] [[package]] name = "rustfs-mimalloc-sys" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692" +checksum = "1dd3ec9b7e7b9fed453acd8b4e32713ddc0e01e1aee31a3deec5b8025c0880c3" dependencies = [ "cc", ] @@ -12301,7 +12271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -12485,9 +12455,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -13652,13 +13622,14 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "wnaf" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" dependencies = [ "ff 0.14.0", "group 0.14.0", "hybrid-array", + "primefield", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 644e5b08a..05ddc1110 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,7 +165,7 @@ http-body = "1.1.0" http-body-util = "0.1.5" minlz = "1.2.3" reqwest = "0.13.4" -rustfs-kafka-async = { version = "1.2.0" } +rustfs-kafka-async = { version = "1.3.1" } socket2 = { version = "0.6.5" } tokio = { version = "1.53.1" } tokio-rustls = { default-features = false, version = "0.26.4" } @@ -244,8 +244,8 @@ aws-sdk-kms = { default-features = false, version = "1.117.0" } aws-sdk-s3 = { default-features = false, version = "1.144.0" } aws-sdk-sts = { default-features = false, version = "1.113.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } -aws-smithy-runtime-api = { version = "1.15.0" } -aws-smithy-types = { version = "1.6.2" } +aws-smithy-runtime-api = { version = "1.16.0" } +aws-smithy-types = { version = "1.6.3" } base64-simd = "0.8.0" brotli = "9.0.0" clap = { version = "4.6.6" } @@ -359,15 +359,15 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "11.0.0" } rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.63.1" } +russh = { version = "0.63.2" } russh-sftp = "2.4.0" # WebDAV dav-server = "0.11.0" # Performance Analysis and Memory Profiling -rustfs-mimalloc = { version = "0.5.2" } -hotpath = { version = "0.24.0", default-features = false } +rustfs-mimalloc = { version = "0.5.3" } +hotpath = { version = "0.25.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index aa04369a6..7ce3523fd 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -252,6 +252,13 @@ pub mod bucket { }; } + pub mod sealed_credentials { + pub use crate::bucket::sealed_credentials::{ + CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError, + SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret, + }; + } + pub mod replication { pub use crate::bucket::replication::replication_pool::{ DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary, diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 7a8b65f75..7649e3ac9 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -87,16 +87,12 @@ use rustfs_filemeta::{ use rustfs_scanner_metrics::metrics::{ IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics, }; -use rustfs_utils::{ - get_env_i64, get_env_usize, - path::encode_dir_object, - string::{parse_bool, strings_has_prefix_fold}, -}; +use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool}; use s3s::dto::{ BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, Timestamp, }; -use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; +use s3s::header::X_AMZ_RESTORE; use sha2::{Digest, Sha256}; use std::any::Any; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" )] pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; -pub const AMZ_ENCRYPTION_AES: &str = "AES256"; #[allow( dead_code, reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" @@ -4802,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation reader } -pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result { +/// Resolve the RestoreObject request options. +/// +/// Returns the typed [`StorageError`]: flattening these into an opaque +/// `io::Error` string erased the identity the S3 layer needs to answer +/// InvalidArgument instead of a generic 500 (backlog#2205). +pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result { let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await; let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await; let vid = version_id.trim(); if !vid.is_empty() && vid != NULL_VERSION_ID { if let Err(_err) = Uuid::parse_str(vid) { - return Err(std::io::Error::other( - StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(), - )); + return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string())); } if !versioned && !version_suspended { - return Err(std::io::Error::other( - StorageError::InvalidArgument( - bucket.to_string(), - object.to_string(), - format!("version-id specified {} but versioning is not enabled on {}", vid, bucket), - ) - .to_string(), + return Err(StorageError::InvalidArgument( + bucket.to_string(), + object.to_string(), + format!("version-id specified {vid} but versioning is not enabled on {bucket}"), )); } } @@ -4872,43 +4867,18 @@ pub async fn put_restore_opts( } meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/ - if let Some(type_) = &rreq.type_ - && type_.as_str() == RestoreRequestType::SELECT + // A SELECT restore must never reach the restore writer: the caller writes + // the retrieved bytes back to the source bucket/object, so building + // SELECT output options here produced a source overwrite carrying only + // the OutputLocation metadata instead of a write to `OutputLocation.S3` + // (backlog#1341). RestoreObject rejects SELECT at the API boundary; this + // is the fail-closed backstop for any other caller. + if rreq + .type_ + .as_ref() + .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT) { - let Some(s3) = select_restore_s3_location(rreq)? else { - return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests")); - }; - if let Some(user_metadata) = s3.user_metadata.as_ref() { - for metadata in user_metadata { - let name = metadata - .name - .as_deref() - .ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?; - let value = metadata.value.clone().unwrap_or_default(); - if strings_has_prefix_fold(name, "x-amz-meta") { - meta.insert(name.to_string(), value); - } else { - meta.insert(format!("x-amz-meta-{name}"), value); - } - } - } - if let Some(tags) = &s3.tagging { - meta.insert( - AMZ_OBJECT_TAGGING.to_string(), - serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()), - ); - } - if let Some(encryption) = &s3.encryption - && encryption.encryption_type.as_str() != "" - { - meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string()); - } - return Ok(ObjectOptions { - versioned: BucketVersioningSys::prefix_enabled(bucket, object).await, - version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await, - user_defined: meta, - ..Default::default() - }); + return Err(std::io::Error::other("SELECT restore requests are not supported")); } for (k, v) in oi.user_defined.iter() { meta.insert(k.to_string(), v.clone()); diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index 48fce4e02..e93419cf2 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -31,6 +31,7 @@ pub mod policy_sys; pub mod quota; pub mod remote_s3_client; pub mod replication; +pub mod sealed_credentials; pub mod tagging; pub mod target; pub mod utils; diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 584cce320..4782e05f0 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -14,9 +14,9 @@ //! Outbound client for an on-demand migration source bucket. //! -//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared -//! remote builder and exposes the read-only surface the migration path -//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a +//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The +//! S3 backend uses the shared remote builder and exposes the surface the +//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a //! probe for admin validation). Every request carries the //! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and //! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of @@ -511,7 +511,7 @@ pub struct SourceObject { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SourcePage { pub objects: Vec, - /// Rolled-up prefixes, in the local namespace; always empty when the + /// Rolled-up prefixes, in the same namespace as `objects`; always empty when the /// request carried no delimiter. pub common_prefixes: Vec, pub is_truncated: bool, @@ -519,7 +519,8 @@ pub struct SourcePage { } /// One `ListObjectsV2` page request against the source. Keys are given in the -/// local namespace; `SourceClient` maps them through `source_prefix`. +/// local namespace at `SourceClient`, and in the source namespace at +/// `SourceBackend`; `SourceClient` maps them through `source_prefix`. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SourceListRequest<'a> { pub prefix: Option<&'a str>, @@ -575,8 +576,30 @@ impl Intercept for SourceProxyMarkerInterceptor { } } -pub struct SourceClient { +/// Read-only provider operations in the source bucket namespace. +/// +/// Implementations must preserve streaming, honor the requested range and +/// pagination cursor, and classify failures without including credentials. +/// `SourceClient` owns prefix mapping so every provider shares the same local +/// namespace. Continuation tokens are opaque and must never be prefix-mapped. +#[async_trait::async_trait] +pub trait SourceBackend: Send + Sync { + async fn head(&self, key: &str) -> Result; + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result; + async fn list(&self, request: &SourceListRequest<'_>) -> Result; + async fn tagging(&self, key: &str) -> Result, SourceError>; + /// Verify bucket access; `SourceClient` separately probes a filtered listing. + async fn probe(&self) -> Result<(), SourceError>; +} + +/// S3-compatible implementation, including request signing and anti-loop headers. +pub struct S3SourceBackend { client: S3Client, + bucket: String, +} + +pub struct SourceClient { + backend: Box, endpoint: String, bucket: String, source_prefix: Option, @@ -609,7 +632,10 @@ impl SourceClient { fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self { let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build()); Self { - client, + backend: Box::new(S3SourceBackend { + client, + bucket: spec.bucket.clone(), + }), endpoint, bucket: spec.bucket.clone(), source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()), @@ -652,35 +678,15 @@ impl SourceClient { } pub async fn head_object(&self, key: &str) -> Result { - let output = self - .client - .head_object() - .bucket(&self.bucket) - .key(self.source_key(key)) - .send() - .await - .map_err(classify_sdk_error)?; - source_head_from_head_output(output) + self.backend.head(&self.source_key(key)).await } - /// Streams the object; `range` is passed through as an HTTP `Range` - /// header and omitted entirely when `None`. + /// Streams the object, preserving an optional HTTP byte range. pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { - let range = range.map(range_header_value).transpose()?; - let output = self - .client - .get_object() - .bucket(&self.bucket) - .key(self.source_key(key)) - .set_range(range) - .send() - .await - .map_err(classify_sdk_error)?; - source_get_from_output(output) + self.backend.get(&self.source_key(key), range).await } - /// Lists one page under the local `prefix`. Keys are returned in the - /// local namespace; entries outside `source_prefix` are skipped. + /// Lists one page under the local prefix. pub async fn list_objects_v2( &self, prefix: Option<&str>, @@ -696,9 +702,81 @@ impl SourceClient { .await } - /// [`Self::list_objects_v2`] with the delimiter and start-after the - /// list-through merge needs (rustfs/backlog#2164). + /// Maps keys and common prefixes while leaving opaque cursors untouched. pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result { + let prefix = self.source_key(request.prefix.unwrap_or_default()); + let start_after = request.start_after.map(|key| self.source_key(key)); + let mut page = self + .backend + .list(&SourceListRequest { + prefix: Some(&prefix), + start_after: start_after.as_deref(), + ..*request + }) + .await?; + page.objects = page + .objects + .into_iter() + .filter_map(|object| self.local_object(object)) + .collect(); + page.common_prefixes = page + .common_prefixes + .into_iter() + .filter_map(|prefix| self.local_key(&prefix).map(str::to_string)) + .collect(); + Ok(page) + } + + fn local_object(&self, mut object: SourceObject) -> Option { + object.key = self.local_key(&object.key)?.to_string(); + Some(object) + } + + pub async fn get_object_tagging(&self, key: &str) -> Result, SourceError> { + self.backend.tagging(&self.source_key(key)).await + } + + pub async fn probe(&self) -> Result { + self.backend.probe().await?; + let page = self.list_objects_v2(None, None, 1).await?; + Ok(SourceProbe { + sample_object: page.objects.into_iter().next(), + has_more_objects: page.is_truncated, + }) + } +} + +#[async_trait::async_trait] +impl SourceBackend for S3SourceBackend { + async fn head(&self, key: &str) -> Result { + let output = self + .client + .head_object() + .bucket(&self.bucket) + .key(key) + .send() + .await + .map_err(classify_sdk_error)?; + source_head_from_head_output(output) + } + + /// Streams the object; `range` is passed through as an HTTP `Range` + /// header and omitted entirely when `None`. + async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result { + let range = range.map(range_header_value).transpose()?; + let output = self + .client + .get_object() + .bucket(&self.bucket) + .key(key) + .set_range(range) + .send() + .await + .map_err(classify_sdk_error)?; + source_get_from_output(output) + } + + async fn list(&self, request: &SourceListRequest<'_>) -> Result { // `start_after` is silently ignored by S3 once a continuation token is // present; refuse the ambiguous pair rather than list from the wrong // position. @@ -711,9 +789,9 @@ impl SourceClient { .client .list_objects_v2() .bucket(&self.bucket) - .prefix(self.source_key(request.prefix.unwrap_or_default())) + .prefix(request.prefix.unwrap_or_default()) .set_delimiter(request.delimiter.map(str::to_string)) - .set_start_after(request.start_after.map(|after| self.source_key(after))) + .set_start_after(request.start_after.map(str::to_string)) .set_continuation_token(request.continuation_token.map(str::to_string)) .max_keys(request.max_keys) .send() @@ -731,13 +809,13 @@ impl SourceClient { .contents .unwrap_or_default() .into_iter() - .filter_map(|object| self.source_object(object)) + .filter_map(s3_source_object) .collect(); let common_prefixes = output .common_prefixes .unwrap_or_default() .into_iter() - .filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string())) + .filter_map(|prefix| prefix.prefix) .collect(); Ok(SourcePage { @@ -748,48 +826,43 @@ impl SourceClient { }) } - fn source_object(&self, object: SdkObject) -> Option { - let key = self.local_key(object.key.as_deref()?)?.to_string(); - let etag = normalize_etag(object.e_tag); - let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); - Some(SourceObject { - key, - etag, - size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), - last_modified: system_time(object.last_modified), - storage_class: object.storage_class.map(|class| class.as_str().to_string()), - is_multipart_etag, - }) - } - - pub async fn get_object_tagging(&self, key: &str) -> Result, SourceError> { + async fn tagging(&self, key: &str) -> Result, SourceError> { let output = self .client .get_object_tagging() .bucket(&self.bucket) - .key(self.source_key(key)) + .key(key) .send() .await .map_err(classify_sdk_error)?; Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect()) } - /// Admin validation: HeadBucket plus a one-key listing under the prefix. - pub async fn probe(&self) -> Result { + async fn probe(&self) -> Result<(), SourceError> { self.client .head_bucket() .bucket(&self.bucket) .send() .await .map_err(classify_sdk_error)?; - let page = self.list_objects_v2(None, None, 1).await?; - Ok(SourceProbe { - sample_object: page.objects.into_iter().next(), - has_more_objects: page.is_truncated, - }) + Ok(()) } } +fn s3_source_object(object: SdkObject) -> Option { + let key = object.key?; + let etag = normalize_etag(object.e_tag); + let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag); + Some(SourceObject { + key, + etag, + size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0), + last_modified: system_time(object.last_modified), + storage_class: object.storage_class.map(|class| class.as_str().to_string()), + is_multipart_etag, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -1194,6 +1267,68 @@ mod tests { assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri); } + #[tokio::test] + async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() { + let body = r#" +truedata/opaque +data/photos/ +outside/ +"#; + let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await; + let first = client + .list_page(&SourceListRequest { + prefix: Some("photos/"), + delimiter: Some("/"), + start_after: Some("photos/a"), + max_keys: 2, + ..Default::default() + }) + .await + .expect("delimiter listing should succeed"); + assert_eq!(first.common_prefixes, vec!["photos/"]); + assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque")); + let second = client + .list_page(&SourceListRequest { + continuation_token: first.next_continuation_token.as_deref(), + max_keys: 2, + ..Default::default() + }) + .await + .expect("opaque continuation should succeed"); + assert_eq!(second.common_prefixes, first.common_prefixes); + let requests = recorded(&requests); + let query = |request: &RecordedRequest| { + Url::parse(&request.uri) + .expect("request URI") + .query_pairs() + .into_owned() + .collect::>() + }; + let first_query = query(&requests[0]); + assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/")); + assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a")); + assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/")); + let second_query = query(&requests[1]); + assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque")); + assert!(!second_query.contains_key("start-after")); + } + + #[tokio::test] + async fn list_page_rejects_ambiguous_cursor_before_sending() { + let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await; + let err = client + .list_page(&SourceListRequest { + start_after: Some("a"), + continuation_token: Some("opaque"), + max_keys: 1, + ..Default::default() + }) + .await + .expect_err("ambiguous list position must fail"); + assert!(matches!(err, SourceError::Other(_))); + assert!(recorded(&requests).is_empty(), "invalid request must never reach the source"); + } + #[tokio::test] async fn list_objects_v2_rejects_truncated_page_without_token() { let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await; @@ -1357,11 +1492,14 @@ mod tests { fn prefix_client(prefix: Option) -> SourceClient { SourceClient { - client: S3Client::from_conf( - aws_sdk_s3::Config::builder() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .build(), - ), + backend: Box::new(S3SourceBackend { + client: S3Client::from_conf( + aws_sdk_s3::Config::builder() + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) + .build(), + ), + bucket: "bucket".to_string(), + }), endpoint: "https://source.example.com".to_string(), bucket: "bucket".to_string(), source_prefix: prefix.filter(|prefix| !prefix.is_empty()), diff --git a/crates/ecstore/src/bucket/sealed_credentials.rs b/crates/ecstore/src/bucket/sealed_credentials.rs new file mode 100644 index 000000000..46a0f79fb --- /dev/null +++ b/crates/ecstore/src/bucket/sealed_credentials.rs @@ -0,0 +1,357 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Sealed remote credentials shared by the three stores that persist one +//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote +//! tiers (`tier-config.bin`) and on-demand migration sources +//! (`on-demand-migration.json`). +//! +//! The design record is `docs/architecture/remote-credential-sealing-adr.md`. +//! What this module owns: the versioned envelope, the encryption context that +//! binds a ciphertext to the record owning it, the sealer registration point, +//! and the fail-closed error type. What it deliberately does not own: any KMS +//! call (ECStore does not depend on `rustfs-kms`; the binary installs a +//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event +//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and +//! any decision about which stored field a consumer writes. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; +use std::sync::{Arc, OnceLock}; + +/// Envelope format this build writes. A reader accepts only versions it +/// knows; an unknown version is a typed error, never a fallback. +pub const SEALED_CREDENTIAL_VERSION: u8 = 1; + +/// Which store a sealed value belongs to. Part of the encryption context, so +/// a ciphertext cannot be replayed into a different store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SealedCredentialStore { + /// `bucket-targets.json` (replication and other bucket targets). + BucketTargets, + /// `tier-config.bin` (remote tiers). + TierConfig, + /// `on-demand-migration.json` (migration sources). + OnDemandMigration, +} + +impl SealedCredentialStore { + pub fn as_str(self) -> &'static str { + match self { + SealedCredentialStore::BucketTargets => "bucket-targets", + SealedCredentialStore::TierConfig => "tier-config", + SealedCredentialStore::OnDemandMigration => "on-demand-migration", + } + } +} + +/// Identity of the record a secret belongs to: the store, its owner (bucket +/// name, tier name, or target ARN) and the field name. Rendered into the KMS +/// encryption context so a ciphertext moved between buckets, tiers or fields +/// fails to decrypt instead of silently authorizing a different remote. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SealScope { + pub store: SealedCredentialStore, + pub owner: String, + pub field: &'static str, +} + +impl SealScope { + pub fn new(store: SealedCredentialStore, owner: impl Into, field: &'static str) -> Self { + Self { + store, + owner: owner.into(), + field, + } + } + + /// The encryption context handed to the sealer. Keys are stable: they are + /// part of the on-disk contract, because a ciphertext only decrypts under + /// the same context. + pub fn encryption_context(&self) -> BTreeMap { + BTreeMap::from([ + ("rustfs:store".to_string(), self.store.as_str().to_string()), + ("rustfs:owner".to_string(), self.owner.clone()), + ("rustfs:field".to_string(), self.field.to_string()), + ]) + } +} + +/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value +/// is not a secret, but it is noise in a log line and an operator reading one +/// should see the key it is wrapped under, not the bytes. +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SealedCredential { + /// Envelope version; see [`SEALED_CREDENTIAL_VERSION`]. + pub v: u8, + /// KMS master key id the data key is wrapped under. + pub key_id: String, + /// Master key version, when the backend reports one. Carried so the KMS + /// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell + /// stale envelopes apart; nothing here rotates on its own. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key_version: Option, + /// Algorithm label reported by the sealer, for forensics and migration. + pub alg: String, + /// Ciphertext blob as produced by the sealer, base64 (standard, padded) + /// in the JSON stores and raw inside the tier msgpack payload. + pub ct: String, +} + +impl fmt::Debug for SealedCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SealedCredential") + .field("v", &self.v) + .field("key_id", &self.key_id) + .field("key_version", &self.key_version) + .field("alg", &self.alg) + .field("ct", &format_args!("<{} bytes sealed>", self.ct.len())) + .finish() + } +} + +impl SealedCredential { + /// Rejects an envelope this build cannot read. Called before every + /// unseal so an unknown version fails here rather than inside a backend. + pub fn check_version(&self) -> Result<(), SealedCredentialError> { + if self.v == SEALED_CREDENTIAL_VERSION { + Ok(()) + } else { + Err(SealedCredentialError::UnsupportedVersion(self.v)) + } + } +} + +/// Why a seal or unseal did not produce a usable value. Every variant is +/// terminal for the record that carried it: a caller reports the remote as +/// unusable, and never substitutes a default or empty credential. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum SealedCredentialError { + /// No sealer is installed: KMS is not configured, or the process has not + /// finished startup. Reading a sealed record is impossible here. + #[error("no credential sealer is installed")] + NoSealer, + /// The stored envelope is from a newer (or otherwise unknown) format. + #[error("unsupported sealed credential envelope version {0}")] + UnsupportedVersion(u8), + /// The stored bytes are not a well-formed envelope. + #[error("malformed sealed credential: {0}")] + Malformed(String), + /// The sealer refused: wrong encryption context, missing key, revoked + /// access, or a failed authentication tag. + #[error("sealed credential could not be unsealed: {0}")] + Kms(String), +} + +/// The KMS-backed half, installed by the binary at startup. +#[async_trait] +pub trait CredentialSealer: Send + Sync + 'static { + /// Wraps `plaintext` under the scope's encryption context. + async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result; + + /// Unwraps a stored envelope. Must fail when the envelope was sealed + /// under a different scope. + async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result; +} + +static CREDENTIAL_SEALER: OnceLock> = OnceLock::new(); + +/// Installs the process-wide sealer. Returns `false` when one is already +/// installed, matching the other ECStore hooks. +pub fn install_credential_sealer(sealer: Arc) -> bool { + CREDENTIAL_SEALER.set(sealer).is_ok() +} + +/// The installed sealer, or `None` when KMS is not wired. Callers that only +/// need to know whether sealing is possible use this; callers that must have +/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error. +pub fn credential_sealer() -> Option> { + CREDENTIAL_SEALER.get().cloned() +} + +/// Seals one secret field. Fails closed: without a sealer the caller must +/// reject the write rather than persist the secret in clear text after the +/// operator asked for sealing. +pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result { + let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?; + sealer.seal(plaintext, scope).await +} + +/// Unseals one secret field, rejecting an unknown envelope version first. +pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result { + sealed.check_version()?; + let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?; + sealer.unseal(sealed, scope).await +} + +#[cfg(test)] +mod tests { + use super::*; + use parking_lot::Mutex; + use std::collections::BTreeMap; + + fn encode_context(context: &HashMap) -> String { + let ordered = context.iter().collect::>(); + serde_json::to_string(&ordered).expect("context serializes") + } + + /// Stands in for the KMS-backed sealer: records the context it was called + /// with, and refuses a ciphertext presented under a different one. + #[derive(Default)] + struct FakeSealer { + sealed_contexts: Mutex>>, + } + + #[async_trait] + impl CredentialSealer for FakeSealer { + async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result { + let context = scope.encryption_context(); + self.sealed_contexts.lock().push(context.clone()); + let mut bound = encode_context(&context); + bound.push('|'); + bound.push_str(plaintext); + Ok(SealedCredential { + v: SEALED_CREDENTIAL_VERSION, + key_id: "key-1".to_string(), + key_version: Some("3".to_string()), + alg: "AES-256-GCM".to_string(), + ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()), + }) + } + + async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result { + let raw = base64_simd::STANDARD + .decode_to_vec(sealed.ct.as_bytes()) + .map_err(|err| SealedCredentialError::Malformed(err.to_string()))?; + let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?; + let expected = encode_context(&scope.encryption_context()); + bound + .strip_prefix(&expected) + .and_then(|rest| rest.strip_prefix('|')) + .map(str::to_string) + .ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string())) + } + } + + fn scope(owner: &str) -> SealScope { + SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key") + } + + #[tokio::test] + async fn seal_round_trips_and_binds_the_scope() { + let sealer = Arc::new(FakeSealer::default()); + let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal"); + assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION); + assert_eq!(sealed.key_version.as_deref(), Some("3")); + assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret"); + + // The same ciphertext under another bucket must not unseal. + let err = sealer + .unseal(&sealed, &scope("other-bucket")) + .await + .expect_err("a ciphertext must not move between owners"); + assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}"); + + // Nor under another field of the same record. + let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token"); + let err = sealer + .unseal(&sealed, &other_field) + .await + .expect_err("a ciphertext must not move between fields"); + assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}"); + + let contexts = sealer.sealed_contexts.lock(); + assert_eq!(contexts.len(), 1); + assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration"); + assert_eq!(contexts[0]["rustfs:owner"], "photos"); + assert_eq!(contexts[0]["rustfs:field"], "secret_key"); + } + + #[tokio::test] + async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() { + let sealed = SealedCredential { + v: SEALED_CREDENTIAL_VERSION + 1, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + assert_eq!( + sealed.check_version().expect_err("a newer envelope must not be read"), + SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1) + ); + // The global helper reports the version, not "no sealer", even in a + // process where none is installed. + assert_eq!( + unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"), + SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1) + ); + } + + #[tokio::test] + async fn without_a_sealer_both_directions_fail_closed() { + // This test binary installs no sealer, so the global helpers must + // report NoSealer rather than fall back to clear text. + assert!(credential_sealer().is_none(), "no sealer is installed in unit tests"); + assert_eq!( + seal_secret("super-secret", &scope("photos")).await.expect_err("seal"), + SealedCredentialError::NoSealer + ); + let sealed = SealedCredential { + v: SEALED_CREDENTIAL_VERSION, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + assert_eq!( + unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"), + SealedCredentialError::NoSealer + ); + } + + #[test] + fn debug_and_serde_keep_the_on_disk_shape_stable() { + let sealed = SealedCredential { + v: 1, + key_id: "key-1".to_string(), + key_version: None, + alg: "AES-256-GCM".to_string(), + ct: "Zm9v".to_string(), + }; + // key_version is omitted when absent, so an envelope from a backend + // without version history stays compact. + assert_eq!( + serde_json::to_string(&sealed).expect("serialize"), + r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"# + ); + let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#) + .expect("an envelope without key_version parses"); + assert_eq!(parsed, sealed); + + let rendered = format!("{sealed:?}"); + assert!(rendered.contains("key-1"), "{rendered}"); + assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}"); + } + + #[test] + fn a_malformed_envelope_is_a_typed_error() { + let err = serde_json::from_str::(r#"{"v":1,"key_id":"key-1"}"#) + .map_err(|err| SealedCredentialError::Malformed(err.to_string())) + .expect_err("a truncated envelope must not parse"); + assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}"); + } +} diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 377f6ba06..887015a1e 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -20,7 +20,7 @@ use byteorder::{ByteOrder, LittleEndian}; use bytes::Bytes; -use futures::FutureExt; +use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; use http::HeaderMap; use http::status::StatusCode; use lazy_static::lazy_static; @@ -50,9 +50,12 @@ use tracing::{debug, error, info, warn}; use crate::error::{Error, Result, StorageError, stable_io_error}; use crate::services::tier::{ tier_admin::TierCreds, - tier_config::{TierConfig, TierType, TierWasabi}, + tier_config::{TIER_CREDENTIAL_REDACTED, TierConfig, TierType, TierWasabi}, tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME}, - warm_backend::{TransitionCandidateProbe, WarmBackend, check_warm_backend, new_warm_backend}, + warm_backend::{ + TransitionCandidateProbe, WARM_BACKEND_PROBE_TIMEOUT, WarmBackend, check_warm_backend, check_warm_backend_until, + new_warm_backend, + }, }; use crate::storage_api_contracts::{ bucket::BucketOperations, @@ -100,6 +103,9 @@ use super::{ const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30); +const TIER_MUTATION_PEER_RPC_TIMEOUT: Duration = Duration::from_secs(30); +const TIER_MUTATION_PEER_FANOUT_TIMEOUT: Duration = Duration::from_secs(30); +const TIER_MUTATION_PEER_FANOUT_CONCURRENCY: usize = 4; const TIER_REFERENCE_PROOF_LIST_LIMIT: i32 = 1000; const TIER_REFERENCE_PROOF_PHYSICAL_WALK_CONCURRENCY: usize = 4; const TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT: usize = 1000; @@ -136,6 +142,19 @@ struct TierDriverBuildBarrier { #[cfg(test)] static TIER_DRIVER_BUILD_BARRIER: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); +#[cfg(test)] +type TierDriverTestFactory = Arc std::result::Result + Send + Sync + 'static>; + +#[cfg(test)] +tokio::task_local! { + static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory; +} + +#[cfg(test)] +tokio::task_local! { + static TIER_REFERENCE_PROOF_TEST_BARRIER: Arc; +} + #[cfg(test)] struct TierDriverBuildBarrierGuard; @@ -157,12 +176,27 @@ fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc std::result::Result { +#[cfg(test)] +fn tier_reference_proof_test_barrier() -> Arc { + Arc::new(TierDriverBuildBarrier { + tier_name: String::new(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Semaphore::new(0), + }) +} + +fn tier_validation_timeout(message: impl Into) -> AdminError { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = message.into(); + err +} + +#[cfg(test)] +async fn wait_for_tier_driver_build_barrier(tier_name: &str) { #[cfg(test)] let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() }; - #[cfg(test)] if let Some(barrier) = test_barrier - && barrier.tier_name == tier.name + && barrier.tier_name == tier_name { barrier.arrived.notify_one(); barrier @@ -172,7 +206,52 @@ async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Resu .expect("tier driver build test barrier should stay open") .forget(); } - new_warm_backend(tier, probe).await +} + +async fn construct_warm_backend(tier: &TierConfig) -> std::result::Result { + #[cfg(test)] + if let Ok(result) = TIER_DRIVER_TEST_FACTORY.try_with(|factory| factory(tier)) { + return result; + } + new_warm_backend(tier, false).await +} + +async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result { + build_warm_backend_with_deadline(tier, probe, None).await +} + +async fn build_warm_backend_with_deadline( + tier: &TierConfig, + probe: bool, + deadline: Option, +) -> std::result::Result { + #[cfg(test)] + { + let wait = wait_for_tier_driver_build_barrier(&tier.name); + if let Some(deadline) = deadline { + timeout_at(deadline, wait) + .await + .map_err(|_| tier_validation_timeout("Timed out preparing the remote tier backend"))?; + } else { + wait.await; + } + } + + let driver = if let Some(deadline) = deadline { + timeout_at(deadline, construct_warm_backend(tier)) + .await + .map_err(|_| tier_validation_timeout("Timed out preparing the remote tier backend"))?? + } else { + construct_warm_backend(tier).await? + }; + if probe { + if let Some(deadline) = deadline { + check_warm_backend_until(Some(&driver), deadline).await?; + } else { + check_warm_backend(Some(&driver)).await?; + } + } + Ok(driver) } const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json"; @@ -219,6 +298,189 @@ lazy_static! { }; } +fn tier_invalid_config(message: impl Into) -> AdminError { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = message.into(); + err +} + +fn normalize_add_tier_name_fields( + canonical_name: &mut String, + nested_name: &mut String, + provider: &str, +) -> std::result::Result<(), AdminError> { + if !canonical_name.is_empty() && !nested_name.is_empty() && canonical_name.as_str() != nested_name.as_str() { + return Err(tier_invalid_config(format!( + "TierConfig.Name conflicts with the legacy {provider}.name field" + ))); + } + let resolved_name = if canonical_name.is_empty() { + nested_name.clone() + } else { + canonical_name.clone() + }; + if resolved_name.is_empty() { + return Err(tier_invalid_config("Remote tier name is empty")); + } + canonical_name.clone_from(&resolved_name); + nested_name.clone_from(&resolved_name); + Ok(()) +} + +fn normalize_s3_gcs_add_tier_name(config: &mut TierConfig) -> std::result::Result<(), AdminError> { + match config.tier_type { + TierType::S3 => { + if let Some(s3) = config.s3.as_mut() { + normalize_add_tier_name_fields(&mut config.name, &mut s3.name, "S3")?; + } + } + TierType::GCS => { + if let Some(gcs) = config.gcs.as_mut() { + normalize_add_tier_name_fields(&mut config.name, &mut gcs.name, "GCS")?; + } + } + _ => {} + } + Ok(()) +} + +fn credential_is_redacted(value: &str) -> bool { + value.trim() == TIER_CREDENTIAL_REDACTED +} + +fn validate_static_tier_credentials(access_key: &str, secret_key: &str) -> std::result::Result<(), AdminError> { + if access_key.is_empty() || secret_key.is_empty() || credential_is_redacted(access_key) || credential_is_redacted(secret_key) + { + return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + } + Ok(()) +} + +fn tier_creds_request_aws_role(creds: &TierCreds) -> bool { + creds.aws_role || !creds.aws_role_web_identity_token_file.is_empty() || !creds.aws_role_arn.is_empty() +} + +fn s3_config_requests_aws_role(config: &crate::services::tier::tier_config::TierS3) -> bool { + config.aws_role + || !config.aws_role_web_identity_token_file.is_empty() + || !config.aws_role_arn.is_empty() + || !config.aws_role_session_name.is_empty() + || config.aws_role_duration_seconds != 0 +} + +fn reject_unsupported_aws_role() -> AdminError { + tier_invalid_config("AWS role and web identity credentials are not supported for remote tiers") +} + +fn validate_gcs_credentials_json(credentials: &str) -> std::result::Result<(), AdminError> { + if credentials.is_empty() || credential_is_redacted(credentials) { + return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + } + let value: serde_json::Value = serde_json::from_str(credentials) + .map_err(|_| tier_invalid_config("GCS credentials must be valid service account JSON"))?; + if value.get("type").and_then(serde_json::Value::as_str) != Some("service_account") { + return Err(tier_invalid_config("GCS credentials must describe a service account")); + } + Ok(()) +} + +fn validate_azure_supported_options( + azure: &crate::services::tier::tier_config::TierAzure, +) -> std::result::Result<(), AdminError> { + let sp_auth_set = + !azure.sp_auth.tenant_id.is_empty() || !azure.sp_auth.client_id.is_empty() || !azure.sp_auth.client_secret.is_empty(); + if !azure.storage_class.is_empty() || sp_auth_set { + return Err(tier_invalid_config( + "Azure remote tiers do not support storageClass or spAuth yet; leave both unset", + )); + } + Ok(()) +} + +fn validate_tier_config_credentials(config: &TierConfig) -> std::result::Result<(), AdminError> { + match config.tier_type { + TierType::S3 => { + if let Some(s3) = config.s3.as_ref() { + if s3_config_requests_aws_role(s3) { + return Err(reject_unsupported_aws_role()); + } + validate_static_tier_credentials(&s3.access_key, &s3.secret_key)?; + } + } + TierType::Wasabi => { + if let Some(backend) = config.wasabi.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::RustFS => { + if let Some(backend) = config.rustfs.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::MinIO => { + if let Some(backend) = config.minio.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Aliyun => { + if let Some(backend) = config.aliyun.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Tencent => { + if let Some(backend) = config.tencent.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Huaweicloud => { + if let Some(backend) = config.huaweicloud.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Azure => { + if let Some(backend) = config.azure.as_ref() { + validate_azure_supported_options(backend)?; + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::GCS => { + if let Some(gcs) = config.gcs.as_ref() { + validate_gcs_credentials_json(&gcs.creds)?; + } + } + TierType::R2 => { + if let Some(backend) = config.r2.as_ref() { + validate_static_tier_credentials(&backend.access_key, &backend.secret_key)?; + } + } + TierType::Unsupported => {} + } + Ok(()) +} + +fn merge_static_tier_credentials( + access_key: &mut String, + secret_key: &mut String, + creds: &TierCreds, +) -> std::result::Result<(), AdminError> { + if tier_creds_request_aws_role(creds) { + return Err(reject_unsupported_aws_role()); + } + if !creds.creds_json.is_empty() { + return Err(tier_invalid_config("GCS credentials cannot be used with this remote tier type")); + } + match (creds.access_key.is_empty(), creds.secret_key.is_empty()) { + (true, true) => {} + (false, false) => { + validate_static_tier_credentials(&creds.access_key, &creds.secret_key)?; + access_key.clone_from(&creds.access_key); + secret_key.clone_from(&creds.secret_key); + } + _ => return Err(ERR_TIER_MISSING_CREDENTIALS.clone()), + } + validate_static_tier_credentials(access_key, secret_key) +} + #[derive(Serialize, Deserialize)] pub struct TierConfigMgr { #[serde(skip)] @@ -323,7 +585,11 @@ struct PreparedTierDriver { impl TierPublishTransition { async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> { - let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT; + self.wait_for_active_leases_until(Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT) + .await + } + + async fn wait_for_active_leases_until(&self, deadline: Instant) -> std::result::Result<(), AdminError> { for generation in self.revoked.values() { if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); @@ -539,15 +805,40 @@ enum TierCandidateMutation { Edit(String, TierCreds), Remove(String, bool), Clear(bool), + Prevalidated(Box), +} + +struct PrevalidatedTierCandidateMutation { + candidate: TierConfigMgr, + current: TierConfigMgr, + version: Option, + kind: TierMutationIntentKind, + explicit_tier_name: Option, + force: bool, + driver_tier: Option, + candidate_digest: TierMutationDigest, } impl TierCandidateMutation { + fn add(mut config: TierConfig, force: bool) -> std::result::Result { + normalize_s3_gcs_add_tier_name(&mut config)?; + Ok(Self::Add(config, force)) + } + + fn normalize_add_tier_name(&mut self) -> std::result::Result<(), AdminError> { + if let Self::Add(config, _) = self { + normalize_s3_gcs_add_tier_name(config)?; + } + Ok(()) + } + fn intent_kind(&self) -> TierMutationIntentKind { match self { Self::Add(_, _) => TierMutationIntentKind::Add, Self::Edit(_, _) => TierMutationIntentKind::Edit, Self::Remove(_, _) => TierMutationIntentKind::Remove, Self::Clear(_) => TierMutationIntentKind::Clear, + Self::Prevalidated(prepared) => prepared.kind, } } @@ -558,6 +849,7 @@ impl TierCandidateMutation { match self { Self::Add(_, force) | Self::Remove(_, force) | Self::Clear(force) => *force, Self::Edit(_, _) => false, + Self::Prevalidated(prepared) => prepared.force, } } @@ -566,6 +858,7 @@ impl TierCandidateMutation { Self::Add(config, _) => Some(&config.name), Self::Edit(tier_name, _) | Self::Remove(tier_name, _) => Some(tier_name), Self::Clear(_) => None, + Self::Prevalidated(prepared) => prepared.explicit_tier_name.as_deref(), } } @@ -586,6 +879,9 @@ impl TierCandidateMutation { Self::Clear(_) => { targets.extend(manager.tiers.keys().chain(candidate.tiers.keys()).cloned()); } + Self::Prevalidated(_) => { + targets.extend(changed_tier_names(manager, candidate)); + } } targets } @@ -605,15 +901,19 @@ impl TierCandidateMutation { ) } - async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result, AdminError> { + async fn apply( + self, + candidate: &mut TierConfigMgr, + deadline: Option, + ) -> std::result::Result, AdminError> { match self { Self::Add(config, force) => { let tier_name = config.name.clone(); - candidate.add(config, force).await?; + candidate.add_with_deadline(config, force, deadline).await?; Ok(Some(tier_name)) } Self::Edit(tier_name, credentials) => { - candidate.edit(&tier_name, credentials).await?; + candidate.edit_with_deadline(&tier_name, credentials, deadline).await?; Ok(Some(tier_name)) } Self::Remove(tier_name, force) => { @@ -624,6 +924,7 @@ impl TierCandidateMutation { candidate.clear_tier(force).await?; Ok(None) } + Self::Prevalidated(_) => unreachable!("prevalidated tier mutation must bypass backend validation"), } } } @@ -714,6 +1015,7 @@ fn tier_backend_identity_admin_error(err: io::Error) -> AdminError { #[async_trait::async_trait] trait TierReferenceProofStore: EcstoreObjectIO + + NamespaceLocking + BucketOperations + ListOperations< Error = Error, @@ -802,12 +1104,24 @@ where { let targets = affected_targets .iter() - .filter(|target| target.old_backend_identity.is_some()) + .filter(|target| target.old_backend_identity.is_some() && target.old_backend_identity != target.new_backend_identity) .cloned() .collect::>(); if targets.is_empty() { return Ok(()); } + #[cfg(test)] + let test_barrier = TIER_REFERENCE_PROOF_TEST_BARRIER.try_with(Arc::clone).ok(); + #[cfg(test)] + if let Some(barrier) = test_barrier { + barrier.arrived.notify_one(); + barrier + .release + .acquire() + .await + .expect("tier reference proof test semaphore should stay open") + .forget(); + } ensure_no_authoritative_target_references(api, &targets, force).await } @@ -1301,44 +1615,63 @@ async fn prepare_tier_mutation_peers( error: tier_mutation_fanout_admin_error("prepare", err), prepared_peers: Vec::new(), })?); + let peers = peers.into_iter().map(|peer| (peer.peer_label(), peer)).collect::>(); + let fanout_deadline = tier_mutation_peer_fanout_deadline(Some(intent.expires_at_unix_nanos)); + let (started, mut outcomes) = run_bounded_tier_mutation_peer_calls(&peers, fanout_deadline, |index, peer| { + let payload = payload.clone(); + async move { + let deadline = Instant::now() + TIER_MUTATION_PEER_RPC_TIMEOUT; + let result = timeout_at(deadline, peer.prepare_tier_mutation(mutation_id, payload)).await; + (index, result) + } + .boxed() + }) + .await; let mut prepared = Vec::with_capacity(peers.len()); - for peer in peers { - let label = peer.peer_label(); - let result = peer.prepare_tier_mutation(mutation_id, payload.clone()).await; - match result { - Ok(PeerTierMutationState::Prepared) => prepared.push(peer), - Ok(PeerTierMutationState::Committed) => {} - Ok(state) => { - return Err(TierMutationPrepareFailure { - error: tier_mutation_fanout_admin_error( - "prepare", - format!("peer {label} returned unexpected state {state:?}"), - ), - prepared_peers: prepared, - }); - } - Err(err) => { - if tier_mutation_prepare_was_definitely_rejected(&err) { - return Err(TierMutationPrepareFailure { - error: tier_mutation_fanout_admin_error("prepare", format!("peer {label}: {err}")), - prepared_peers: prepared, - }); + let mut failures = Vec::new(); + for (index, (label, peer)) in peers.into_iter().enumerate() { + match outcomes[index].take() { + Some(result) => match result { + Ok(Ok(PeerTierMutationState::Prepared)) => prepared.push(peer), + Ok(Ok(PeerTierMutationState::Committed)) => {} + Ok(Ok(state)) => failures.push(format!("peer {label} returned unexpected state {state:?}")), + Ok(Err(err)) => { + if tier_mutation_prepare_was_definitely_rejected(&err) { + failures.push(format!("peer {label}: {err}")); + continue; + } + // The request may have reached the peer and installed its + // durable Prepared block before the response failed (for + // example while draining an in-flight lease). Include it in + // abort fanout so ambiguous prepare outcomes fail closed + // without stranding a runtime block. + prepared.push(peer); + failures.push(format!("peer {label}: {err}")); } - // The request may have reached the peer and installed its - // durable Prepared block before the response failed (for - // example while draining an in-flight lease). Include it in - // abort fanout so ambiguous prepare outcomes fail closed - // without stranding a runtime block. + Err(_) => { + prepared.push(peer); + failures.push(format!("peer {label}: request timed out")); + } + }, + None if started[index] => { + // The fanout-wide deadline cancelled an in-flight request. + // Its durable result is unknown, so compensating Abort must + // include this peer. prepared.push(peer); - return Err(TierMutationPrepareFailure { - error: tier_mutation_fanout_admin_error("prepare", format!("peer {label}: {err}")), - prepared_peers: prepared, - }); + failures.push(format!("peer {label}: fanout deadline expired with request in flight")); } + None => failures.push(format!("peer {label}: fanout deadline expired before request start")), } } - Ok(prepared) + if failures.is_empty() { + Ok(prepared) + } else { + Err(TierMutationPrepareFailure { + error: tier_mutation_fanout_admin_error("prepare", failures.join("; ")), + prepared_peers: prepared, + }) + } } fn tier_mutation_prepare_was_definitely_rejected(err: &Error) -> bool { @@ -1350,19 +1683,84 @@ struct TierMutationPrepareFailure { prepared_peers: Vec>, } +type TierMutationPeerCallOutcome = std::result::Result, tokio::time::error::Elapsed>; + +fn tier_mutation_peer_fanout_deadline(intent_expiry_unix_nanos: Option) -> Instant { + let mut budget = TIER_MUTATION_PEER_FANOUT_TIMEOUT; + if let Some(expires_at) = intent_expiry_unix_nanos { + let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); + let remaining = i128::from(expires_at).saturating_sub(now); + let remaining = if remaining <= 0 { + Duration::ZERO + } else { + Duration::from_nanos(u64::try_from(remaining).unwrap_or(u64::MAX)) + }; + budget = budget.min(remaining); + } + Instant::now() + budget +} + +async fn run_bounded_tier_mutation_peer_calls( + peers: &[(String, Arc)], + fanout_deadline: Instant, + make_call: F, +) -> (Vec, Vec>) +where + F: Fn(usize, Arc) -> futures::future::BoxFuture<'static, (usize, TierMutationPeerCallOutcome)>, +{ + let mut started = vec![false; peers.len()]; + let mut outcomes = std::iter::repeat_with(|| None).take(peers.len()).collect::>(); + let mut pending = FuturesUnordered::new(); + let mut next = 0; + while next < peers.len() && pending.len() < TIER_MUTATION_PEER_FANOUT_CONCURRENCY && Instant::now() < fanout_deadline { + started[next] = true; + pending.push(make_call(next, peers[next].1.clone())); + next += 1; + } + while !pending.is_empty() { + let completed = match timeout_at(fanout_deadline, pending.next()).await { + Ok(Some(completed)) => completed, + Ok(None) | Err(_) => break, + }; + outcomes[completed.0] = Some(completed.1); + if next < peers.len() && Instant::now() < fanout_deadline { + started[next] = true; + pending.push(make_call(next, peers[next].1.clone())); + next += 1; + } + } + (started, outcomes) +} + async fn abort_tier_mutation_peers(intent: &TierMutationIntent, peers: Vec>) -> io::Result<()> { let prepared = intent.original_prepared().map_err(io::Error::other)?; let payload = Bytes::from(prepared.encode().map_err(io::Error::other)?); + let mutation_id = intent.mutation_id; + let peers = peers.into_iter().map(|peer| (peer.peer_label(), peer)).collect::>(); + let fanout_deadline = tier_mutation_peer_fanout_deadline(None); + let (started, mut outcomes) = run_bounded_tier_mutation_peer_calls(&peers, fanout_deadline, |index, peer| { + let payload = payload.clone(); + async move { + let deadline = Instant::now() + TIER_MUTATION_PEER_RPC_TIMEOUT; + let result = timeout_at(deadline, peer.abort_tier_mutation(mutation_id, payload)).await; + (index, result) + } + .boxed() + }) + .await; let mut failures = Vec::new(); - for peer in peers { - let label = peer.peer_label(); - let result = peer.abort_tier_mutation(intent.mutation_id, payload.clone()).await; - match result { - Ok(PeerTierMutationState::Aborted) => {} - Ok(state) => { - failures.push(format!("peer {label} returned unexpected abort state {state:?}")); - } - Err(err) => failures.push(format!("peer {label}: {err}")), + for (index, (label, _)) in peers.iter().enumerate() { + match outcomes[index].take() { + Some(result) => match result { + Ok(Ok(PeerTierMutationState::Aborted)) => {} + Ok(Ok(state)) => { + failures.push(format!("peer {label} returned unexpected abort state {state:?}")); + } + Ok(Err(err)) => failures.push(format!("peer {label}: {err}")), + Err(_) => failures.push(format!("peer {label}: request timed out")), + }, + None if started[index] => failures.push(format!("peer {label}: fanout deadline expired with request in flight")), + None => failures.push(format!("peer {label}: fanout deadline expired before request start")), } } if failures.is_empty() { @@ -1378,18 +1776,36 @@ async fn commit_tier_mutation_peers( committed_config_etag: &str, ) -> io::Result<()> { let payload = Bytes::copy_from_slice(committed_config_etag.as_bytes()); - for peer in peers { - let label = peer.peer_label(); - let result = peer.commit_tier_mutation(mutation_id, payload.clone()).await; - match result { - Ok(PeerTierMutationState::Committed) => {} - Ok(state) => { - return Err(tier_mutation_replay_error(format!("peer {label} returned unexpected state {state:?}"))); - } - Err(err) => return Err(tier_mutation_replay_error(format!("peer {label}: {err}"))), + let peers = peers.into_iter().map(|peer| (peer.peer_label(), peer)).collect::>(); + let fanout_deadline = tier_mutation_peer_fanout_deadline(None); + let (started, mut outcomes) = run_bounded_tier_mutation_peer_calls(&peers, fanout_deadline, |index, peer| { + let payload = payload.clone(); + async move { + let deadline = Instant::now() + TIER_MUTATION_PEER_RPC_TIMEOUT; + let result = timeout_at(deadline, peer.commit_tier_mutation(mutation_id, payload)).await; + (index, result) + } + .boxed() + }) + .await; + let mut failures = Vec::new(); + for (index, (label, _)) in peers.iter().enumerate() { + match outcomes[index].take() { + Some(result) => match result { + Ok(Ok(PeerTierMutationState::Committed)) => {} + Ok(Ok(state)) => failures.push(format!("peer {label} returned unexpected state {state:?}")), + Ok(Err(err)) => failures.push(format!("peer {label}: {err}")), + Err(_) => failures.push(format!("peer {label}: request timed out")), + }, + None if started[index] => failures.push(format!("peer {label}: fanout deadline expired with request in flight")), + None => failures.push(format!("peer {label}: fanout deadline expired before request start")), } } - Ok(()) + if failures.is_empty() { + Ok(()) + } else { + Err(tier_mutation_replay_error(failures.join("; "))) + } } fn tier_config_digest(bytes: &[u8]) -> TierMutationDigest { @@ -1572,7 +1988,12 @@ async fn apply_tier_candidate_mutation( candidate: &mut TierConfigMgr, deadline: Instant, ) -> std::result::Result, AdminError> { - match timeout_at(deadline, mutation.apply(candidate)).await { + if matches!(&mutation, TierCandidateMutation::Add(_, _) | TierCandidateMutation::Edit(_, _)) { + // Add/Edit validation performs its own deadline-aware cleanup. Do not + // wrap it in an outer timeout that would cancel an uncertain probe. + return mutation.apply(candidate, Some(deadline)).await; + } + match timeout_at(deadline, mutation.apply(candidate, None)).await { Ok(result) => result, Err(_) => { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); @@ -1582,6 +2003,45 @@ async fn apply_tier_candidate_mutation( } } +async fn prevalidate_tier_candidate_mutation( + mut candidate: TierConfigMgr, + version: Option, + mutation: TierCandidateMutation, +) -> std::result::Result { + let kind = mutation.intent_kind(); + if version.is_none() && !candidate.tiers.is_empty() && kind != TierMutationIntentKind::Add { + return Err(TierConfigUpdateError::Load(io::Error::other( + "tier configuration mutation requires an existing config ETag", + ))); + } + let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string); + let force = mutation.force(); + let current = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: candidate + .tiers + .iter() + .map(|(tier_name, config)| (tier_name.clone(), config.clone_with_credentials())) + .collect(), + last_refreshed_at: candidate.last_refreshed_at, + }; + let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline) + .await + .map_err(TierConfigUpdateError::Mutation)?; + let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?; + Ok(PrevalidatedTierCandidateMutation { + candidate, + current, + version, + kind, + explicit_tier_name, + force, + driver_tier, + candidate_digest, + }) +} + fn changed_tier_names(current: &TierConfigMgr, replacement: &TierConfigMgr) -> HashSet { let mut changed = HashSet::new(); for (tier_name, current_config) in ¤t.tiers { @@ -2157,6 +2617,22 @@ struct ExternalTierCompatible { region: String, } +fn decode_external_gcs_credentials(credentials: &str) -> io::Result { + if serde_json::from_str::(credentials).is_ok() { + return Ok(credentials.to_string()); + } + // MinIO config and madmin AddTier payloads carry URL-safe-base64 credentials, + // while the existing RustFS v2 disk format stores raw JSON for rolling upgrades. + let decoded = base64_simd::STANDARD + .decode_to_vec(credentials.as_bytes()) + .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(credentials.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(credentials.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(credentials.as_bytes())) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "tier config contains invalid GCS credentials encoding"))?; + String::from_utf8(decoded) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "tier config contains non-UTF-8 GCS credentials JSON")) +} + fn tier_config_path(file: &str) -> String { format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file) } @@ -2577,7 +3053,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul cfg.gcs = Some(crate::services::tier::tier_config::TierGCS { name: cfg.name.clone(), endpoint: gcs.endpoint.clone(), - creds: gcs.creds.clone(), + creds: decode_external_gcs_credentials(&gcs.creds)?, bucket: gcs.bucket.clone(), prefix: gcs.prefix.clone(), region: gcs.region.clone(), @@ -2765,7 +3241,17 @@ impl TierConfigMgr { (TierType::Unsupported, false) } - pub async fn add(&mut self, mut tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + pub async fn add(&mut self, tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + self.add_with_deadline(tier_config, force, None).await + } + + async fn add_with_deadline( + &mut self, + mut tier_config: TierConfig, + force: bool, + deadline: Option, + ) -> std::result::Result<(), AdminError> { + normalize_s3_gcs_add_tier_name(&mut tier_config)?; self.ensure_generation_is_idle(&tier_config.name)?; let tier_name = tier_config.name.clone(); if tier_name != tier_name.to_uppercase() { @@ -2797,28 +3283,28 @@ impl TierConfigMgr { })?; } - // The Azure warm backend goes through the same S3-compatible TransitionClient as every - // other provider (backlog#2055): it has no Azure Blob-native client and no Azure AD - // dependency, so `storage_class` and `sp_auth` cannot be honored today even though the - // config type carries them. Reject them explicitly here instead of silently accepting - // and then dropping them at the WarmBackendAzure construction boundary. - if matches!(&tier_config.tier_type, TierType::Azure) - && let Some(azure) = tier_config.azure.as_ref() + if matches!(&tier_config.tier_type, TierType::GCS) + && let Some(gcs) = tier_config.gcs.as_mut() { - let sp_auth_set = !azure.sp_auth.tenant_id.is_empty() - || !azure.sp_auth.client_id.is_empty() - || !azure.sp_auth.client_secret.is_empty(); - if !azure.storage_class.is_empty() || sp_auth_set { - let mut err = ERR_TIER_INVALID_CONFIG.clone(); - err.message = "Azure remote tiers do not support storageClass or spAuth yet; leave both unset".to_string(); - return Err(err); - } + gcs.creds = decode_external_gcs_credentials(&gcs.creds).map_err(|source| tier_invalid_config(source.to_string()))?; } - let d = new_warm_backend(&tier_config, true).await?; + validate_tier_config_credentials(&tier_config)?; + let d = match deadline { + Some(deadline) => build_warm_backend_with_deadline(&tier_config, true, Some(deadline)).await?, + None => build_warm_backend(&tier_config, true).await?, + }; if !force { - let in_use = d.in_use().await; + let in_use = match deadline { + Some(deadline) => timeout_at(deadline, d.in_use()).await, + None => tokio::time::timeout(WARM_BACKEND_PROBE_TIMEOUT, d.in_use()).await, + } + .map_err(|_| { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out checking whether the remote tier is in use".to_string(); + err + })?; match in_use { Ok(b) => { if b { @@ -2910,7 +3396,7 @@ impl TierConfigMgr { pub fn list_tiers(&self) -> Vec { let mut tier_cfgs = Vec::::new(); for (_, tier) in self.tiers.iter() { - let tier = tier.clone(); + let tier = tier.redacted(); tier_cfgs.push(tier); } tier_cfgs @@ -2919,120 +3405,108 @@ impl TierConfigMgr { pub fn get(&self, tier_name: &str) -> Option { for (tier_name2, tier) in self.tiers.iter() { if tier_name == tier_name2 { - return Some(tier.clone()); + return Some(tier.redacted()); } } None } pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> { + self.edit_with_deadline(tier_name, creds, None).await + } + + async fn edit_with_deadline( + &mut self, + tier_name: &str, + creds: TierCreds, + deadline: Option, + ) -> std::result::Result<(), AdminError> { self.ensure_generation_is_idle(tier_name)?; let (tier_type, exists) = self.is_tier_name_in_use(tier_name); if !exists { return Err(ERR_TIER_NOT_FOUND.clone()); } + if !creds.azure_service_principal.is_empty() { + return Err(tier_invalid_config( + "Azure service principal credentials are not supported for remote tier edits", + )); + } - let mut tier_config = self.tiers[tier_name].clone(); + let mut tier_config = self.tiers[tier_name].clone_with_credentials(); match tier_type { TierType::S3 => { if let Some(s3) = tier_config.s3.as_mut() { - if creds.aws_role { - s3.aws_role = true - } - if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" { - s3.aws_role_arn = creds.aws_role_arn; - s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file; - } - if creds.access_key != "" && creds.secret_key != "" { - s3.access_key = creds.access_key; - s3.secret_key = creds.secret_key; - } + merge_static_tier_credentials(&mut s3.access_key, &mut s3.secret_key, &creds)?; } } TierType::Wasabi => { if let Some(wasabi) = tier_config.wasabi.as_mut() { - if creds.access_key.is_empty() || creds.secret_key.is_empty() { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - wasabi.access_key = creds.access_key; - wasabi.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut wasabi.access_key, &mut wasabi.secret_key, &creds)?; } } TierType::RustFS => { if let Some(rustfs) = tier_config.rustfs.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - rustfs.access_key = creds.access_key; - rustfs.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut rustfs.access_key, &mut rustfs.secret_key, &creds)?; } } TierType::MinIO => { if let Some(compatible_backend) = tier_config.minio.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - compatible_backend.access_key = creds.access_key; - compatible_backend.secret_key = creds.secret_key; + merge_static_tier_credentials( + &mut compatible_backend.access_key, + &mut compatible_backend.secret_key, + &creds, + )?; } } TierType::Aliyun => { if let Some(aliyun) = tier_config.aliyun.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - aliyun.access_key = creds.access_key; - aliyun.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut aliyun.access_key, &mut aliyun.secret_key, &creds)?; } } TierType::Tencent => { if let Some(tencent) = tier_config.tencent.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - tencent.access_key = creds.access_key; - tencent.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut tencent.access_key, &mut tencent.secret_key, &creds)?; } } TierType::Huaweicloud => { if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - huaweicloud.access_key = creds.access_key; - huaweicloud.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut huaweicloud.access_key, &mut huaweicloud.secret_key, &creds)?; } } TierType::Azure => { if let Some(azure) = tier_config.azure.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - azure.access_key = creds.access_key; - azure.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut azure.access_key, &mut azure.secret_key, &creds)?; } } TierType::GCS => { if let Some(gcs) = tier_config.gcs.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); + if tier_creds_request_aws_role(&creds) { + return Err(reject_unsupported_aws_role()); + } + if !creds.access_key.is_empty() || !creds.secret_key.is_empty() { + return Err(tier_invalid_config("Static access and secret keys cannot be used with a GCS tier")); + } + if !creds.creds_json.is_empty() { + let credentials = std::str::from_utf8(&creds.creds_json) + .map_err(|_| tier_invalid_config("GCS credentials must be UTF-8 service account JSON"))?; + validate_gcs_credentials_json(credentials)?; + gcs.creds = credentials.to_string(); } - gcs.creds = creds.access_key; //creds.creds_json } } TierType::R2 => { if let Some(r2) = tier_config.r2.as_mut() { - if creds.access_key == "" || creds.secret_key == "" { - return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); - } - r2.access_key = creds.access_key; - r2.secret_key = creds.secret_key; + merge_static_tier_credentials(&mut r2.access_key, &mut r2.secret_key, &creds)?; } } _ => (), } - let d = new_warm_backend(&tier_config, true).await?; + validate_tier_config_credentials(&tier_config)?; + let d = match deadline { + Some(deadline) => build_warm_backend_with_deadline(&tier_config, true, Some(deadline)).await?, + None => build_warm_backend(&tier_config, true).await?, + }; self.revoke_driver(tier_name); self.tiers.insert(tier_name.to_string(), tier_config); self.replace_driver(tier_name, d)?; @@ -3266,18 +3740,101 @@ impl TierConfigMgr { async fn update_candidate_with_config_lock( handle: &Arc>, api: Arc, - mutation: TierCandidateMutation, + mut mutation: TierCandidateMutation, ) -> std::result::Result<(), TierConfigUpdateError> where S: TierReferenceProofStore + NamespaceLocking + 'static, { + mutation.normalize_add_tier_name().map_err(TierConfigUpdateError::Mutation)?; + let initial_config_lock = Self::acquire_tier_config_write_lock(api.clone()).await?; + Self::reject_pending_mutation_recovery_before_update(handle, api.clone()).await?; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + drop(initial_config_lock); + + let prepared = Self::prevalidate_candidate_owned(candidate, version, mutation).await?; + let config_lock = Self::acquire_tier_config_write_lock(api.clone()).await?; Self::reject_pending_mutation_recovery_before_update(handle, api.clone()).await?; let update = Self::admin_update_lock(handle).await; let (candidate, version) = load_tier_config_for_update(api.clone()) .await .map_err(TierConfigUpdateError::Load)?; - Self::update_candidate_owned(handle, api, candidate, version, mutation, update, Some(config_lock)).await + if version != prepared.version { + return Err(TierConfigUpdateError::Load(io::Error::other( + "tier configuration changed while the remote backend mutation was being validated", + ))); + } + Self::update_candidate_owned( + handle, + api, + candidate, + version, + TierCandidateMutation::Prevalidated(Box::new(prepared)), + update, + Some(config_lock), + ) + .await + } + + async fn prevalidate_candidate_owned( + candidate: TierConfigMgr, + version: Option, + mutation: TierCandidateMutation, + ) -> std::result::Result { + #[cfg(test)] + let test_driver_factory = TIER_DRIVER_TEST_FACTORY.try_with(|factory| factory.clone()).ok(); + tokio::spawn(async move { + let validation = prevalidate_tier_candidate_mutation(candidate, version, mutation); + #[cfg(test)] + if let Some(factory) = test_driver_factory { + return TIER_DRIVER_TEST_FACTORY.scope(factory, validation).await; + } + validation.await + }) + .await + .map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = format!("Remote tier backend validation task failed: {err}"); + TierConfigUpdateError::Mutation(admin_err) + })? + } + + async fn revalidate_prepared_candidate_before_commit( + api: Arc, + version: Option<&str>, + candidate_digest: TierMutationDigest, + coordinator_intent: Option<&TierMutationIntent>, + ) -> io::Result<()> + where + S: TierReferenceProofStore, + { + let (_, current_version) = load_tier_config_for_update(api.clone()).await?; + if current_version.as_deref() != version { + return Err(io::Error::other( + "tier configuration changed while the prepared mutation reference proof was running", + )); + } + let Some(expected) = coordinator_intent else { + return Ok(()); + }; + if expected.candidate_digest != candidate_digest { + return Err(io::Error::other("prepared tier mutation candidate digest changed before commit")); + } + let (persisted, _) = load_tier_coordinator_mutation_intent_record_with_etag(api, expected.mutation_id) + .await + .map_err(io::Error::other)?; + if persisted.state != TierMutationIntentState::Prepared || !persisted.same_identity_as(expected) { + return Err(io::Error::other( + "prepared tier mutation intent changed while the reference proof was running", + )); + } + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX); + if persisted.expires_at_unix_nanos <= now { + return Err(io::Error::other("prepared tier mutation intent expired before config commit")); + } + Ok(()) } async fn reject_pending_mutation_recovery_before_update( @@ -3452,7 +4009,7 @@ impl TierConfigMgr { let exact_get_delete = tier_exact_get_delete(config); Some(PreparedTierDriver { tier_name: tier_name.to_string(), - tier_config: config.clone(), + tier_config: config.clone_with_credentials(), config_fingerprint, backend_identity, exact_get_delete, @@ -3493,7 +4050,7 @@ impl TierConfigMgr { })?; let entry = Arc::new(TierDriverGeneration { tier_name: Arc::from(prepared.tier_name.as_str()), - tier_config: prepared.tier_config.clone(), + tier_config: prepared.tier_config.clone_with_credentials(), generation, config_fingerprint: prepared.config_fingerprint, backend_identity: prepared.backend_identity, @@ -3596,42 +4153,96 @@ impl TierConfigMgr { let handle = handle.clone(); #[cfg(test)] let test_peers = TIER_MUTATION_TEST_PEERS.try_with(|peers| peers.clone()).ok(); + #[cfg(test)] + let test_driver_factory = TIER_DRIVER_TEST_FACTORY.try_with(|factory| factory.clone()).ok(); + #[cfg(test)] + let test_reference_proof_barrier = TIER_REFERENCE_PROOF_TEST_BARRIER.try_with(Arc::clone).ok(); tokio::spawn(async move { let update_task = async move { match AssertUnwindSafe(async move { - let _config_lock = config_lock; - let _update = update; - let mutation_kind = mutation.intent_kind(); - if version.is_none() && !candidate.tiers.is_empty() && mutation_kind != TierMutationIntentKind::Add { - return Err(TierConfigUpdateError::Load(io::Error::other( - "tier configuration mutation requires an existing config ETag", - ))); - } - let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string); - let mutation_force = mutation.force(); - let current_for_targets = TierConfigMgr { - driver_cache: HashMap::new(), - tiers: candidate - .tiers - .iter() - .map(|(tier_name, config)| (tier_name.clone(), config.clone_with_credentials())) - .collect(), - last_refreshed_at: candidate.last_refreshed_at, - }; - let mut transition = { - let mut manager = handle.write().await; - let target_tiers = mutation.target_tiers(&manager, &candidate); - Self::begin_tier_transition(&handle, &mut manager, target_tiers) - .map_err(TierConfigUpdateError::Publish)? - }; - transition - .wait_for_active_leases() - .await - .map_err(TierConfigUpdateError::Publish)?; - let driver_tier = - apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT) - .await - .map_err(TierConfigUpdateError::Mutation)?; + let mut config_lock = config_lock; + let coordinated_config_update = config_lock.is_some(); + let mut update = Some(update); + let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) = + match mutation { + TierCandidateMutation::Prevalidated(prepared) => { + if version != prepared.version { + return Err(TierConfigUpdateError::Load(io::Error::other( + "tier configuration changed after remote backend validation", + ))); + } + if tier_config_candidate_digest(&prepared.candidate).map_err(TierConfigUpdateError::Save)? + != prepared.candidate_digest + { + return Err(TierConfigUpdateError::Save(io::Error::other( + "prevalidated tier mutation candidate digest changed", + ))); + } + candidate = prepared.candidate; + let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + let mut transition = { + let mut manager = handle.write().await; + let mut target_tiers = changed_tier_names(&manager, &candidate); + if let Some(tier_name) = prepared.explicit_tier_name.as_ref() + && (manager.tiers.contains_key(tier_name) + || prepared.current.tiers.contains_key(tier_name) + || candidate.tiers.contains_key(tier_name)) + { + target_tiers.insert(tier_name.clone()); + } + Self::begin_tier_transition(&handle, &mut manager, target_tiers) + .map_err(TierConfigUpdateError::Publish)? + }; + ( + prepared.kind, + prepared.explicit_tier_name, + prepared.force, + prepared.current, + prepared.driver_tier, + transition, + ) + } + mutation => { + let mutation_kind = mutation.intent_kind(); + if version.is_none() + && !candidate.tiers.is_empty() + && mutation_kind != TierMutationIntentKind::Add + { + return Err(TierConfigUpdateError::Load(io::Error::other( + "tier configuration mutation requires an existing config ETag", + ))); + } + let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string); + let mutation_force = mutation.force(); + let current_for_targets = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: candidate + .tiers + .iter() + .map(|(tier_name, config)| (tier_name.clone(), config.clone_with_credentials())) + .collect(), + last_refreshed_at: candidate.last_refreshed_at, + }; + let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + let mut transition = { + let mut manager = handle.write().await; + let target_tiers = mutation.target_tiers(&manager, &candidate); + Self::begin_tier_transition(&handle, &mut manager, target_tiers) + .map_err(TierConfigUpdateError::Publish)? + }; + let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline) + .await + .map_err(TierConfigUpdateError::Mutation)?; + ( + mutation_kind, + explicit_tier_name, + mutation_force, + current_for_targets, + driver_tier, + transition, + ) + } + }; let proof_targets = tier_mutation_proof_targets( mutation_kind, explicit_tier_name.as_deref(), @@ -3656,6 +4267,24 @@ impl TierConfigMgr { .await .map_err(TierConfigUpdateError::Publish)?; } + if coordinated_config_update { + drop(update.take()); + drop(config_lock.take()); + } + let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await { + if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation lease drain failed and abort was incomplete" + ); + } + return Err(TierConfigUpdateError::Publish(drain_error)); + } let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() { let peers = match remote_tier_mutation_peers().await { Ok(peers) => peers, @@ -3722,6 +4351,66 @@ impl TierConfigMgr { return Err(TierConfigUpdateError::Publish(proof_error)); } let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?; + if coordinated_config_update { + config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await { + Ok(config_lock) => Some(config_lock), + Err(lock_error) => { + if !abort_prepared_tier_mutation( + &handle, + api.clone(), + coordinator_intent.as_ref(), + prepared_peers, + ) + .await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation config-lock reacquisition failed and abort was incomplete" + ); + } + return Err(lock_error); + } + }; + update = Some(Self::admin_update_lock(&handle).await); + if let Err(revalidation_error) = Self::revalidate_prepared_candidate_before_commit( + api.clone(), + version.as_deref(), + candidate_digest, + coordinator_intent.as_ref(), + ) + .await + { + drop(update.take()); + drop(config_lock.take()); + if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers) + .await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation revalidation failed and abort was incomplete" + ); + } + return Err(TierConfigUpdateError::Save(revalidation_error)); + } + } + if update.is_none() { + return Err(TierConfigUpdateError::Save(io::Error::other( + "tier mutation lost local serialization before config commit", + ))); + } + if coordinated_config_update && config_lock.is_none() { + return Err(TierConfigUpdateError::Save(io::Error::other( + "tier mutation lost namespace serialization before config commit", + ))); + } let saved = match candidate .save_tiering_config_if_current_with_info(api.clone(), version.as_deref()) .await @@ -3734,6 +4423,8 @@ impl TierConfigMgr { ..Default::default() }, Ok(false) => { + drop(update.take()); + drop(config_lock.take()); let aborted = abort_prepared_tier_mutation( &handle, api.clone(), @@ -3795,6 +4486,10 @@ impl TierConfigMgr { commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag) .await .map_err(TierConfigUpdateError::Save)?; + if coordinated_config_update { + drop(update.take()); + drop(config_lock.take()); + } if let Some(intent) = committed_coordinator_intent.as_ref() && !prepared_peers.is_empty() { @@ -3830,11 +4525,17 @@ impl TierConfigMgr { }; #[cfg(test)] { - if let Some(peers) = test_peers { - TIER_MUTATION_TEST_PEERS.scope(peers, update_task).await - } else { - update_task.await + let mut update_task = update_task.boxed(); + if let Some(barrier) = test_reference_proof_barrier { + update_task = TIER_REFERENCE_PROOF_TEST_BARRIER.scope(barrier, update_task).boxed(); } + if let Some(peers) = test_peers { + update_task = TIER_MUTATION_TEST_PEERS.scope(peers, update_task).boxed(); + } + if let Some(factory) = test_driver_factory { + update_task = TIER_DRIVER_TEST_FACTORY.scope(factory, update_task).boxed(); + } + update_task.await } #[cfg(not(test))] { @@ -4326,6 +5027,14 @@ impl TierConfigMgr { continue; } + // The old config remaining current does not prove that a + // Prepared mutation was abandoned: its live coordinator may be + // performing peer fanout or the reference proof without holding + // the namespace lock. Preserve every unexpired durable fence. + if intent.expires_at_unix_nanos > now { + continue; + } + let peers = match &peers { Some(peers) => peers.clone(), None => { @@ -4334,14 +5043,9 @@ impl TierConfigMgr { resolved } }; - let recovery = if intent.expires_at_unix_nanos <= now { - "expired" - } else { - "proven-old" - }; abort_tier_mutation_peers(intent, peers) .await - .map_err(|err| io::Error::other(format!("{recovery} coordinator tier mutation recovery abort failed: {err}")))?; + .map_err(|err| io::Error::other(format!("expired coordinator tier mutation recovery abort failed: {err}")))?; let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await; let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent( api.clone(), @@ -4923,7 +5627,8 @@ impl TierConfigMgr { tier_config: TierConfig, force: bool, ) -> std::result::Result<(), TierConfigUpdateError> { - Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Add(tier_config, force)).await + let mutation = TierCandidateMutation::add(tier_config, force).map_err(TierConfigUpdateError::Mutation)?; + Self::update_candidate_with_config_lock(handle, api, mutation).await } pub async fn edit_and_save( @@ -4936,6 +5641,32 @@ impl TierConfigMgr { .await } + #[cfg(test)] + async fn edit_and_save_with( + handle: &Arc>, + api: Arc, + tier_name: &str, + credentials: TierCreds, + ) -> std::result::Result<(), TierConfigUpdateError> + where + S: TierReferenceProofStore + 'static, + { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned( + handle, + api, + candidate, + version, + TierCandidateMutation::Edit(tier_name.to_string(), credentials), + update, + None, + ) + .await + } + pub async fn remove_and_save( handle: &Arc>, api: Arc, @@ -4999,8 +5730,14 @@ impl TierConfigMgr { let lease = Self::acquire_operation_lease(handle, tier_name) .await .map_err(io::Error::other)?; - let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone())); - check_warm_backend(Some(&driver)).await.map_err(io::Error::other) + tokio::spawn(async move { + let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone())); + let result = check_warm_backend(Some(&driver)).await.map_err(io::Error::other); + drop(lease); + result + }) + .await + .map_err(|_| io::Error::other("remote tier verification task failed"))? } pub(crate) async fn acquire_operation_lease_for_backend_identity( @@ -5060,7 +5797,7 @@ impl TierConfigMgr { let driver: SharedWarmBackend = Arc::from(driver); let entry = Arc::new(TierDriverGeneration { tier_name: Arc::from(tier_name), - tier_config: config.clone(), + tier_config: config.clone_with_credentials(), generation, config_fingerprint, backend_identity, @@ -6379,7 +7116,7 @@ mod tests { // // These tests must not reach a real remote tier. Two techniques keep them // hermetic: - // * error paths that return *before* `new_warm_backend` constructs a + // * error paths that return *before* `build_warm_backend` constructs a // client (name validation, duplicate detection, unsupported type, // missing backend payload, missing credentials); // * a `MockWarmBackend` injected directly into `driver_cache`, so @@ -6387,6 +7124,7 @@ mod tests { // lets us drive `remove`/`verify` through every branch. // --------------------------------------------------------------------- + use crate::services::tier::test_util::{MockWarmBackend as RecordingWarmBackend, MockWarmOp}; use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts}; use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; @@ -6417,6 +7155,24 @@ mod tests { } } + fn build_gcs_tier(name: &str, credentials: &str) -> TierConfig { + TierConfig { + version: "v1".to_string(), + tier_type: TierType::GCS, + name: name.to_string(), + gcs: Some(crate::services::tier::tier_config::TierGCS { + name: name.to_string(), + endpoint: "https://storage.googleapis.com".to_string(), + creds: credentials.to_string(), + bucket: "bucket-gcs".to_string(), + prefix: "prefix-gcs".to_string(), + region: String::new(), + storage_class: String::new(), + }), + ..Default::default() + } + } + #[derive(Debug)] struct LockingTierConfigStore { locks: Mutex>, @@ -6718,7 +7474,12 @@ mod tests { .await .expect("empty clear should still acquire and save through the coordinator lock"); - assert_eq!(store.lock_events(), vec![(RUSTFS_META_BUCKET.to_string(), tier_config_lock_path())]); + let expected_lock = (RUSTFS_META_BUCKET.to_string(), tier_config_lock_path()); + assert_eq!( + store.lock_events(), + vec![expected_lock.clone(), expected_lock.clone(), expected_lock], + "the update should lock for the validation snapshot, durable Prepare, and config commit" + ); assert!( store.put_after_lock.load(Ordering::SeqCst), "tier config save must happen after the coordinator namespace lock is acquired" @@ -6766,6 +7527,51 @@ mod tests { in_use_value: Option, /// When false, put/get/remove all fail (drives `verify` error paths). healthy: bool, + probe_present: AtomicBool, + } + + #[derive(Default)] + struct HangingInUseBackend { + probe_present: AtomicBool, + } + + #[async_trait::async_trait] + impl WarmBackend for HangingInUseBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result { + self.probe_present.store(true, Ordering::SeqCst); + Ok("probe-version".to_string()) + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> io::Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result { + Ok(BufReader::new(Cursor::new(b"RustFS".to_vec()))) + } + + async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> { + self.probe_present.store(false, Ordering::SeqCst); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> io::Result { + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent("probe-version".to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> io::Result { + std::future::pending().await + } } #[async_trait::async_trait] @@ -6788,6 +7594,7 @@ mod tests { async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result { if self.healthy { + self.probe_present.store(true, Ordering::SeqCst); Ok("mock-version".to_string()) } else { Err(std::io::Error::other("mock put failed")) @@ -6819,6 +7626,7 @@ mod tests { async fn remove(&self, _object: &str, _rv: &str) -> std::result::Result<(), std::io::Error> { if self.healthy { + self.probe_present.store(false, Ordering::SeqCst); Ok(()) } else { Err(std::io::Error::other("mock remove failed")) @@ -6832,6 +7640,20 @@ mod tests { self.remove(object, rv).await } + async fn probe_transition_candidate( + &self, + _object: &str, + ) -> std::result::Result { + if !self.healthy { + return Err(std::io::Error::other("mock candidate probe failed")); + } + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent("mock-version".to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> std::result::Result { match self.in_use_value { Some(b) => Ok(b), @@ -6845,8 +7667,37 @@ mod tests { mgr.driver_cache.insert(name.to_string(), Box::new(mock)); } + fn recording_driver_factory( + backend: RecordingWarmBackend, + observed_configs: Arc>>, + ) -> TierDriverTestFactory { + Arc::new(move |config| { + lock_unpoisoned(&observed_configs).push(config.clone_with_credentials()); + Ok(Box::new(backend.clone())) + }) + } + + fn healthy_driver_factory() -> TierDriverTestFactory { + recording_driver_factory(RecordingWarmBackend::new(), Arc::new(Mutex::new(Vec::new()))) + } + // ---- add ------------------------------------------------------------ + #[test] + fn test_add_name_normalization_uses_canonical_name_and_supports_legacy_nested_name() { + let mut canonical_s3 = build_s3_tier("COLD-CANONICAL"); + canonical_s3.s3.as_mut().expect("S3 payload should exist").name.clear(); + normalize_s3_gcs_add_tier_name(&mut canonical_s3).expect("canonical S3 name should normalize"); + assert_eq!(canonical_s3.name, "COLD-CANONICAL"); + assert_eq!(canonical_s3.s3.as_ref().expect("S3 payload should remain").name, "COLD-CANONICAL"); + + let mut legacy_gcs = build_gcs_tier("COLD-LEGACY", r#"{"type":"service_account"}"#); + legacy_gcs.name.clear(); + normalize_s3_gcs_add_tier_name(&mut legacy_gcs).expect("legacy nested GCS name should normalize"); + assert_eq!(legacy_gcs.name, "COLD-LEGACY"); + assert_eq!(legacy_gcs.gcs.as_ref().expect("GCS payload should remain").name, "COLD-LEGACY"); + } + #[tokio::test] async fn test_add_rejects_non_uppercase_name() { let mut mgr = empty_mgr(); @@ -6952,24 +7803,102 @@ mod tests { assert!(mgr.tiers.is_empty()); } + #[tokio::test] + async fn test_add_rejects_canonical_azure_sp_auth_wire_before_backend_setup() { + let tier: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "azure", + "Name": "COLD-AZURE", + "azure": { + "name": "COLD-AZURE", + "endpoint": "https://azure.example.invalid", + "accessKey": "account", + "secretKey": "key", + "bucket": "archive", + "spAuth": { + "TenantID": "tenant" + } + } + })) + .expect("mixed RustFS/madmin Azure payload should decode"); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let mut mgr = empty_mgr(); + + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(tier, true)) + .await + .expect_err("canonical Azure service-principal fields must fail closed"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("spAuth"), "{}", err.message); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0); + assert!(mgr.tiers.is_empty()); + } + #[tokio::test] async fn test_add_does_not_reject_azure_config_without_storage_class_or_sp_auth() { // A plain Azure config (the common case: static access/secret key, no - // storageClass, no spAuth) must sail past the new gate. `new_warm_backend` - // builds the S3-compatible client lazily (no eager DNS/connect), so with - // `force: true` (which also skips the `in_use` probe) this succeeds even - // against a fake endpoint — the point here is only that the gate itself - // does not fire. + // storageClass, no spAuth) must sail past the provider-specific gate. let mut mgr = empty_mgr(); let tier = build_azure_tier("account-a"); let tier_name = tier.name.clone(); - mgr.add(tier, true) + TIER_DRIVER_TEST_FACTORY + .scope(healthy_driver_factory(), mgr.add(tier, true)) .await .expect("a config with no storageClass/spAuth must not trip the new gate"); assert!(mgr.tiers.contains_key(&tier_name)); } + #[tokio::test] + async fn test_add_force_does_not_bypass_bad_credentials_or_unreachable_probe() { + for unreachable in [false, true] { + let mut mgr = empty_mgr(); + let backend = RecordingWarmBackend::new(); + if unreachable { + backend.set_unreachable(true).await; + } else { + backend.set_reject_credentials(true).await; + } + let observed = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, observed.clone()), + mgr.add(build_s3_tier("COLD-A"), true), + ) + .await + .expect_err("force must not bypass the backend read/write/delete probe"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(lock_unpoisoned(&observed).len(), 1); + assert!(mgr.tiers.is_empty()); + } + } + + #[tokio::test] + async fn test_add_rejects_unsupported_s3_role_before_backend_setup() { + let mut mgr = empty_mgr(); + let mut tier = build_s3_tier("COLD-A"); + let s3 = tier.s3.as_mut().expect("S3 payload should exist"); + s3.access_key.clear(); + s3.secret_key.clear(); + s3.aws_role = true; + + let err = mgr + .add(tier, true) + .await + .expect_err("unsupported role credentials must fail before backend setup"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + assert!(mgr.tiers.is_empty()); + } + #[tokio::test] async fn test_add_rejects_reserved_names() { // Supersedes the former `test_add_does_not_reserve_standard_name_regression_anchor` @@ -6998,6 +7927,240 @@ mod tests { } } + #[tokio::test] + async fn test_add_rejects_canonical_s3_role_fields_even_with_static_credentials() { + let mut tier: TierConfig = serde_json::from_value(serde_json::json!({ + "Version": "v1", + "Type": "s3", + "Name": "COLD-A", + "S3": { + "Endpoint": "https://s3.example.invalid", + "AccessKey": "static-access", + "SecretKey": "static-secret", + "Bucket": "archive", + "Prefix": "objects", + "Region": "us-east-1", + "StorageClass": "STANDARD", + "AWSRole": true, + "AWSRoleWebIdentityTokenFile": "/var/run/private-token", + "AWSRoleARN": "arn:aws:iam::123456789012:role/archive", + "AWSRoleSessionName": "archive-session", + "AWSRoleDurationSeconds": 900 + } + })) + .expect("canonical madmin S3 role payload should decode"); + assert_eq!(tier.name, "COLD-A"); + let s3 = tier.s3.as_ref().expect("S3 payload should decode"); + assert!(s3.name.is_empty(), "canonical madmin S3 has no nested Name field"); + assert_eq!(s3.endpoint, "https://s3.example.invalid"); + assert!(!s3.access_key.is_empty()); + assert!(!s3.secret_key.is_empty()); + assert_eq!(s3.bucket, "archive"); + assert_eq!(s3.prefix, "objects"); + assert_eq!(s3.region, "us-east-1"); + assert_eq!(s3.storage_class, "STANDARD"); + assert!(s3.aws_role); + assert_eq!(s3.aws_role_web_identity_token_file, "/var/run/private-token"); + assert_eq!(s3.aws_role_duration_seconds, 900); + normalize_s3_gcs_add_tier_name(&mut tier).expect("canonical madmin Name should normalize before validation"); + assert_eq!(tier.name, "COLD-A"); + assert_eq!(tier.s3.as_ref().expect("S3 payload should remain").name, "COLD-A"); + let serialized = serde_json::to_value(&tier).expect("S3 config should serialize"); + assert_eq!(serialized["type"], "s3"); + assert!(serialized.get("Type").is_none()); + assert!(serialized.get("S3").is_none()); + assert!(serialized.get("Name").is_none()); + assert!(serialized["s3"].get("AccessKey").is_none()); + assert!(serialized["s3"].get("accessKey").is_some()); + assert!(serialized["s3"].get("AWSRole").is_none()); + assert!(serialized["s3"].get("AWSRoleWebIdentityTokenFile").is_none()); + + let mut mgr = empty_mgr(); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_backend_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_backend_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(tier, true)) + .await + .expect_err("unsupported role fields must be rejected before backend setup"); + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0); + assert!(mgr.tiers.is_empty()); + } + + #[tokio::test] + async fn test_add_rejects_conflicting_canonical_and_legacy_s3_gcs_names_before_backend_setup() { + let fixtures = [ + ( + "S3", + serde_json::json!({ + "Type": "S3", + "Name": "COLD-CANONICAL", + "S3": { + "name": "COLD-LEGACY", + "Endpoint": "https://s3.example.invalid", + "AccessKey": "access", + "SecretKey": "secret", + "Bucket": "archive" + } + }), + ), + ( + "GCS", + serde_json::json!({ + "Type": "GCS", + "Name": "COLD-CANONICAL", + "GCS": { + "name": "COLD-LEGACY", + "Endpoint": "https://storage.googleapis.com/", + "Creds": "e30=", + "Bucket": "archive" + } + }), + ), + ]; + + for (provider, fixture) in fixtures { + let tier: TierConfig = serde_json::from_value(fixture) + .unwrap_or_else(|err| panic!("{provider} aliases should decode before name validation: {err}")); + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let backend_builds = Arc::new(AtomicUsize::new(0)); + let observed_backend_builds = backend_builds.clone(); + let factory: TierDriverTestFactory = Arc::new(move |_| { + observed_backend_builds.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(RecordingWarmBackend::new())) + }); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + factory, + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::update_candidate_with_config_lock(&manager, store, TierCandidateMutation::Add(tier, true)), + ), + ) + .await + .expect_err("conflicting canonical and legacy tier names must fail closed"); + let TierConfigUpdateError::Mutation(err) = err else { + panic!("conflicting {provider} names should fail as a typed mutation error") + }; + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("conflicts"), "{}", err.message); + assert_eq!(backend_builds.load(Ordering::SeqCst), 0, "{provider} backend must not be prepared"); + assert!(lock_unpoisoned(&peer_calls).is_empty(), "{provider} conflict must not prepare peers"); + assert!(manager.read().await.tiers.is_empty()); + } + } + + #[tokio::test(start_paused = true)] + async fn test_add_times_out_a_hanging_in_use_check_after_probe_cleanup() { + let mut mgr = empty_mgr(); + let factory: TierDriverTestFactory = Arc::new(|_| Ok(Box::new(HangingInUseBackend::default()))); + let err = TIER_DRIVER_TEST_FACTORY + .scope(factory, mgr.add(build_rustfs_tier("COLD-A"), false)) + .await + .expect_err("a hanging in-use check must not retain the admin lock forever"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out checking")); + assert!(mgr.tiers.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn candidate_add_honors_the_caller_validation_deadline() { + let mut candidate = empty_mgr(); + let factory: TierDriverTestFactory = Arc::new(|_| Ok(Box::new(HangingInUseBackend::default()))); + let deadline = Instant::now() + Duration::from_secs(1); + let err = { + let add = TIER_DRIVER_TEST_FACTORY.scope( + factory, + apply_tier_candidate_mutation( + TierCandidateMutation::Add(build_rustfs_tier("COLD-DEADLINE"), false), + &mut candidate, + deadline, + ), + ); + tokio::pin!(add); + tokio::task::yield_now().await; + + let result = tokio::time::timeout(Duration::from_secs(2), &mut add); + tokio::pin!(result); + tokio::time::advance(Duration::from_secs(2) + Duration::from_millis(1)).await; + result + .await + .expect("the add mutation must finish within its caller deadline") + .expect_err("a hanging in-use check must fail the add mutation") + }; + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out checking")); + assert!(candidate.tiers.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn candidate_edit_honors_the_caller_validation_deadline() { + let mut candidate = empty_mgr(); + candidate + .tiers + .insert("COLD-DEADLINE".to_string(), build_rustfs_tier("COLD-DEADLINE")); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + let factory = recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))); + let deadline = Instant::now() + Duration::from_secs(1); + let err = { + let edit = TIER_DRIVER_TEST_FACTORY.scope( + factory, + apply_tier_candidate_mutation( + TierCandidateMutation::Edit( + "COLD-DEADLINE".to_string(), + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + &mut candidate, + deadline, + ), + ); + tokio::pin!(edit); + tokio::select! { + _ = get_barrier.wait_until_paused() => {} + result = &mut edit => panic!("edit completed before the probe deadline: {result:?}"), + } + + let result = tokio::time::timeout(Duration::from_secs(2), &mut edit); + tokio::pin!(result); + tokio::time::advance(Duration::from_secs(2) + Duration::from_millis(1)).await; + result + .await + .expect("the edit mutation must finish within its caller deadline") + .expect_err("a hanging in-use check must fail the edit mutation") + }; + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("Timed out validating")); + assert_eq!(backend.object_count().await, 0, "deadline-aware edit must clean up its probe object"); + assert_eq!( + candidate.tiers["COLD-DEADLINE"] + .rustfs + .as_ref() + .expect("tier payload") + .access_key, + "ak" + ); + } + // ---- edit ----------------------------------------------------------- #[tokio::test] @@ -7011,32 +8174,46 @@ mod tests { } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_rustfs() { + async fn test_edit_rejects_half_static_credentials_for_rustfs() { let mut mgr = empty_mgr(); mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); - // Empty access/secret keys => rejected before any driver rebuild. let err = mgr - .edit("COLD-R", TierCreds::default()) + .edit( + "COLD-R", + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty credentials must be rejected"); + .expect_err("a half-filled static credential pair must be rejected"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); + let rustfs = mgr.tiers["COLD-R"].rustfs.as_ref().expect("original payload should remain"); + assert_eq!(rustfs.access_key, "ak"); + assert_eq!(rustfs.secret_key, "sk"); } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_wasabi() { + async fn test_edit_rejects_half_static_credentials_for_wasabi() { let mut mgr = empty_mgr(); mgr.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI")); let err = mgr - .edit("COLD-WASABI", TierCreds::default()) + .edit( + "COLD-WASABI", + TierCreds { + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty Wasabi credentials must be rejected before backend setup"); + .expect_err("a half-filled Wasabi credential pair must be rejected before backend setup"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); } #[tokio::test] - async fn test_edit_rejects_missing_credentials_for_minio() { + async fn test_edit_rejects_half_static_credentials_for_minio() { let mut mgr = empty_mgr(); let tier = TierConfig { version: "v1".to_string(), @@ -7056,12 +8233,290 @@ mod tests { mgr.tiers.insert("COLD-M".to_string(), tier); let err = mgr - .edit("COLD-M", TierCreds::default()) + .edit( + "COLD-M", + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + ) .await - .expect_err("empty credentials must be rejected"); + .expect_err("a half-filled static credential pair must be rejected"); assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); } + #[tokio::test] + async fn test_edit_with_omitted_secret_fields_preserves_real_credentials() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); + let backend = RecordingWarmBackend::new(); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))), + mgr.edit("COLD-R", TierCreds::default()), + ) + .await + .expect("an edit with omitted secret fields should validate the preserved credentials"); + + let rustfs = mgr.tiers["COLD-R"].rustfs.as_ref().expect("edited payload should remain"); + assert_eq!(rustfs.access_key, "ak"); + assert_eq!(rustfs.secret_key, "sk"); + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn test_edit_rejects_redaction_placeholder() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R")); + + let err = mgr + .edit( + "COLD-R", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: TIER_CREDENTIAL_REDACTED.to_string(), + ..Default::default() + }, + ) + .await + .expect_err("the API redaction placeholder must never become a real credential"); + + assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code); + assert_eq!( + mgr.tiers["COLD-R"] + .rustfs + .as_ref() + .expect("original payload should remain") + .secret_key, + "sk" + ); + } + + #[tokio::test] + async fn test_edit_rejects_unsupported_s3_role_before_backend_setup() { + let mut mgr = empty_mgr(); + mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A")); + + let err = mgr + .edit( + "COLD-A", + TierCreds { + aws_role: true, + ..Default::default() + }, + ) + .await + .expect_err("unsupported role credentials must fail before backend setup"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("not supported")); + } + + #[tokio::test] + async fn test_edit_gcs_rotates_from_creds_json_and_runs_backend_probe() { + const OLD_CREDS: &str = r#"{"type":"service_account","project_id":"old-project"}"#; + const ROTATED_CREDS: &str = r#"{"type":"service_account","project_id":"rotated-project"}"#; + let mut mgr = empty_mgr(); + mgr.tiers + .insert("COLD-GCS".to_string(), build_gcs_tier("COLD-GCS", OLD_CREDS)); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + mgr.edit( + "COLD-GCS", + TierCreds { + creds_json: ROTATED_CREDS.as_bytes().to_vec(), + ..Default::default() + }, + ), + ) + .await + .expect("GCS service account rotation should validate through the fake backend"); + + assert_eq!( + mgr.tiers["COLD-GCS"].gcs.as_ref().expect("GCS payload should remain").creds, + ROTATED_CREDS + ); + assert_eq!( + lock_unpoisoned(&observed)[0] + .gcs + .as_ref() + .expect("observed GCS payload should exist") + .creds, + ROTATED_CREDS + ); + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn test_gcs_madmin_add_wire_normalizes_url_safe_credentials_to_raw_v2_storage() { + const INITIAL_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; + const INITIAL_MINIO_URL_BASE64: &str = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt4K-_In0="; + const ROTATED_JSON: &str = r#"{"type":"service_account","project_id":"rotated-project"}"#; + let madmin_wire_add: TierConfig = serde_json::from_value(serde_json::json!({ + "Version": "v1", + "Type": "gcs", + "Name": "COLD-GCS", + "GCS": { + "Endpoint": "https://storage.googleapis.com/", + "Creds": INITIAL_MINIO_URL_BASE64, + "Bucket": "bucket-gcs", + "Prefix": "prefix-gcs", + "Region": "", + "StorageClass": "" + } + })) + .expect("canonical madmin GCS AddTier payload should decode"); + assert_eq!(madmin_wire_add.name, "COLD-GCS"); + assert!( + madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration") + .name + .is_empty(), + "canonical madmin GCS has no nested Name field" + ); + assert_eq!( + madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration") + .creds, + INITIAL_MINIO_URL_BASE64 + ); + let gcs = madmin_wire_add + .gcs + .as_ref() + .expect("the decoded madmin payload should contain GCS configuration"); + assert_eq!(gcs.endpoint, "https://storage.googleapis.com/"); + assert_eq!(gcs.bucket, "bucket-gcs"); + assert_eq!(gcs.prefix, "prefix-gcs"); + assert!(gcs.region.is_empty()); + assert!(gcs.storage_class.is_empty()); + let rustfs_output = serde_json::to_value(&madmin_wire_add).expect("GCS config should serialize"); + assert_eq!(rustfs_output["type"], "gcs"); + assert!(rustfs_output.get("Type").is_none()); + assert!(rustfs_output.get("GCS").is_none()); + assert!(rustfs_output.get("Name").is_none()); + assert!(rustfs_output["gcs"].get("Creds").is_none()); + assert_eq!(rustfs_output["gcs"]["creds"], INITIAL_MINIO_URL_BASE64); + let mut mgr = empty_mgr(); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + mgr.add(madmin_wire_add, true), + ) + .await + .expect("GCS add should normalize madmin URL-safe credentials before validation"); + + assert_eq!( + lock_unpoisoned(&observed)[0] + .gcs + .as_ref() + .expect("the backend should receive a GCS payload") + .creds, + INITIAL_JSON + ); + assert_eq!( + mgr.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("the added GCS payload should exist") + .creds, + INITIAL_JSON + ); + assert_eq!(mgr.tiers["COLD-GCS"].name, "COLD-GCS"); + assert_eq!( + mgr.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("the added GCS payload should exist") + .name, + "COLD-GCS" + ); + + let blob = encode_external_tiering_config_blob(&mgr).expect("GCS add should persist externally"); + let external: ExternalTierConfigMgr = rmp_serde::from_slice(&blob[4..]).expect("external GCS payload should decode"); + assert_eq!( + external.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + INITIAL_JSON + ); + let mut loaded = decode_external_tiering_config_blob(&blob).expect("persisted GCS tier should load"); + assert_eq!( + loaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("loaded GCS payload should exist") + .creds, + INITIAL_JSON + ); + + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + loaded.edit( + "COLD-GCS", + TierCreds { + creds_json: ROTATED_JSON.as_bytes().to_vec(), + ..Default::default() + }, + ), + ) + .await + .expect("GCS edit should accept raw madmin credential bytes"); + assert_eq!( + loaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("edited GCS payload should exist") + .creds, + ROTATED_JSON + ); + let rotated_blob = encode_external_tiering_config_blob(&loaded).expect("edited GCS tier should persist externally"); + assert_eq!( + rmp_serde::from_slice::(&rotated_blob[4..]) + .expect("rotated external tier should exist") + .tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("rotated external GCS payload should exist") + .creds, + ROTATED_JSON + ); + let reloaded = decode_external_tiering_config_blob(&rotated_blob).expect("edited GCS tier should reload"); + assert_eq!( + reloaded.tiers["COLD-GCS"] + .gcs + .as_ref() + .expect("reloaded GCS payload should exist") + .creds, + ROTATED_JSON + ); + } + // ---- remove --------------------------------------------------------- #[tokio::test] @@ -7081,6 +8536,7 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7103,6 +8559,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7122,6 +8579,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7140,11 +8598,12 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true); mutation - .apply(&mut candidate) + .apply(&mut candidate, None) .await .expect("force remove must skip in-use probing"); assert!(!candidate.tiers.contains_key("COLD-A")); @@ -7160,10 +8619,11 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); let err = TierCandidateMutation::Remove("COLD-A".to_string(), false) - .apply(&mut candidate) + .apply(&mut candidate, None) .await .expect_err("non-force remove must reject an in-use backend"); assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code); @@ -7180,6 +8640,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); forced.clear_tier(true).await.expect("force clear must skip in-use probing"); @@ -7193,6 +8654,7 @@ mod tests { MockWarmBackend { in_use_value: Some(true), healthy: true, + probe_present: AtomicBool::new(false), }, ); let err = guarded @@ -7213,6 +8675,7 @@ mod tests { MockWarmBackend { in_use_value: None, healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7243,6 +8706,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }, ); @@ -7259,6 +8723,7 @@ mod tests { MockWarmBackend { in_use_value: Some(false), healthy: false, + probe_present: AtomicBool::new(false), }, ); @@ -7272,6 +8737,7 @@ mod tests { let unhealthy: SharedWarmBackend = Arc::new(MockWarmBackend { in_use_value: Some(false), healthy: false, + probe_present: AtomicBool::new(false), }); let proxy = SharedWarmBackendProxy(unhealthy); let err = proxy.validate().await.expect_err("proxy must forward backend validation"); @@ -7280,6 +8746,7 @@ mod tests { let healthy: SharedWarmBackend = Arc::new(MockWarmBackend { in_use_value: Some(false), healthy: true, + probe_present: AtomicBool::new(false), }); let proxy = SharedWarmBackendProxy(healthy); let err = proxy @@ -7329,7 +8796,7 @@ mod tests { assert_eq!(tier.tier_type.as_lowercase(), "s3"); let s3 = tier.s3.as_ref().expect("s3 payload survives"); assert_eq!(s3.bucket, "bucket-a"); - // secret_key is a serialized field (unlike Clone, marshal does not redact). + // Internal serialization preserves credentials; only explicit admin views redact them. assert_eq!(s3.secret_key, "sk"); } @@ -7374,7 +8841,8 @@ mod tests { } #[test] - fn test_external_blob_roundtrip_gcs_preserves_creds() { + fn test_external_blob_gcs_writer_stays_raw_for_v2_old_readers() { + const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; let mut mgr = empty_mgr(); mgr.tiers.insert( "COLD-G".to_string(), @@ -7385,7 +8853,7 @@ mod tests { gcs: Some(crate::services::tier::tier_config::TierGCS { name: "COLD-G".to_string(), endpoint: "https://storage.googleapis.com".to_string(), - creds: "service-account-json".to_string(), + creds: RAW_JSON.to_string(), bucket: "gbucket".to_string(), prefix: "gp".to_string(), region: "us".to_string(), @@ -7396,10 +8864,118 @@ mod tests { ); let bytes = encode_external_tiering_config_blob(&mgr).expect("encode gcs tier"); + assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes()); + assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes()); + let external: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external GCS payload should decode"); + assert_eq!( + external.tiers["COLD-G"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + RAW_JSON + ); + serde_json::from_str::( + &external.tiers["COLD-G"] + .gcs + .as_ref() + .expect("external GCS payload should exist") + .creds, + ) + .expect("the baseline RustFS v2 reader must receive raw GCS credential JSON"); let decoded = decode_external_tiering_config_blob(&bytes).expect("decode gcs tier"); let tier = decoded.tiers.get("COLD-G").expect("gcs tier survives roundtrip"); assert_eq!(tier.tier_type.as_lowercase(), "gcs"); - assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, "service-account-json"); + assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, RAW_JSON); + + let rewritten = encode_external_tiering_config_blob(&decoded).expect("decoded GCS tier should remain v2-compatible"); + let rewritten: ExternalTierConfigMgr = + rmp_serde::from_slice(&rewritten[4..]).expect("rewritten external GCS payload should decode"); + assert_eq!( + rewritten.tiers["COLD-G"] + .gcs + .as_ref() + .expect("rewritten external GCS payload should exist") + .creds, + RAW_JSON + ); + } + + fn decode_hex_fixture(hex: &str) -> Vec { + assert_eq!(hex.len() % 2, 0, "hex fixture must contain complete bytes"); + hex.as_bytes() + .chunks_exact(2) + .map(|pair| { + let pair = std::str::from_utf8(pair).expect("hex fixture should be ASCII"); + u8::from_str_radix(pair, 16).expect("hex fixture should contain only hexadecimal digits") + }) + .collect() + } + + #[test] + fn test_external_blob_decodes_fixed_minio_gcs_fixture_and_rewrites_raw_v2_credentials() { + const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#; + // Produced by MinIO RELEASE.2025-10-15T17-29-55Z with madmin-go/v3 + // v3.0.109. Keeping the bytes fixed prevents this compatibility test + // from accidentally validating RustFS against its own serializer. + const MINIO_GCS_BLOB_HEX: &str = concat!( + "0100020081a5546965727381a6434f4c442d4787a756657273696f6ea27631a45479706503a44e616d65a6434f4c442d47", + "a25333c0a5417a757265c0a347435386a8456e64706f696e74bf68747470733a2f2f73746f726167652e676f6f676c656170", + "69732e636f6d2fa54372656473d94465794a306558426c496a6f6963325679646d6c6a5a56396859324e7664573530496977", + "6963484a76616d566a644639705a434936496e52705a584974344b2d5f496e303da64275636b6574a7676275636b6574a650", + "7265666978a26770a6526567696f6ea27573ac53746f72616765436c617373a84e4541524c494e45a54d696e494fc0" + ); + const MINIO_GCS_BLOB_SHA256_HEX: &str = "bee1d4822d4936bc6f764c284381596d16f2096f40671b25500aca3b710ceac8"; + let fixture = decode_hex_fixture(MINIO_GCS_BLOB_HEX); + assert_eq!(fixture.len(), 246); + assert_eq!( + Sha256::digest(&fixture).as_slice(), + decode_hex_fixture(MINIO_GCS_BLOB_SHA256_HEX).as_slice(), + "fixed foreign fixture digest changed" + ); + + let decoded = decode_external_tiering_config_blob(&fixture) + .expect("the new reader should accept MinIO URL-safe-base64 GCS credentials"); + let gcs = decoded.tiers["COLD-G"] + .gcs + .as_ref() + .expect("decoded MinIO GCS payload should exist"); + assert_eq!(gcs.endpoint, "https://storage.googleapis.com/"); + assert_eq!(gcs.bucket, "gbucket"); + assert_eq!(gcs.prefix, "gp"); + assert_eq!(gcs.region, "us"); + assert_eq!(gcs.storage_class, "NEARLINE"); + assert_eq!(gcs.creds, RAW_JSON); + + let rewritten = encode_external_tiering_config_blob(&decoded) + .expect("MinIO credentials should rewrite in the existing RustFS v2 raw format"); + let rewritten: ExternalTierConfigMgr = + rmp_serde::from_slice(&rewritten[4..]).expect("rewritten MinIO fixture should decode"); + assert_eq!( + rewritten.tiers["COLD-G"] + .gcs + .as_ref() + .expect("rewritten MinIO GCS payload should exist") + .creds, + RAW_JSON + ); + } + + #[test] + fn external_gcs_credentials_accept_all_base64_alphabets_and_padding_modes() { + let raw = r#"{"type":"service_account","project_id":"tier-🚀"}"#; + for encoder in [ + base64_simd::STANDARD, + base64_simd::STANDARD_NO_PAD, + base64_simd::URL_SAFE, + base64_simd::URL_SAFE_NO_PAD, + ] { + let encoded = encoder.encode_to_string(raw.as_bytes()); + assert_eq!( + decode_external_gcs_credentials(&encoded).expect("supported GCS encoding should decode"), + raw + ); + } } // `TierConfigMgr` intentionally does not derive `Debug` (it holds live @@ -7546,6 +9122,7 @@ mod tests { struct LeaseTestBackend { id: &'static str, calls: Arc>>, + probe_present: Arc, remove_started: Option>, remove_release: Option>, backend_in_use: bool, @@ -7558,6 +9135,7 @@ mod tests { Self { id, calls: Arc::new(std::sync::Mutex::new(Vec::new())), + probe_present: Arc::new(AtomicBool::new(false)), remove_started: None, remove_release: None, backend_in_use: false, @@ -7603,6 +9181,7 @@ mod tests { #[async_trait::async_trait] impl WarmBackend for LeaseTestBackend { async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result { + self.probe_present.store(true, Ordering::SeqCst); Ok(self.id.to_string()) } @@ -7617,7 +9196,7 @@ mod tests { } async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result { - Ok(BufReader::new(Cursor::new(Vec::new()))) + Ok(BufReader::new(Cursor::new(b"RustFS".to_vec()))) } async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> { @@ -7635,9 +9214,18 @@ mod tests { .expect("lease test release semaphore should stay open") .forget(); } + self.probe_present.store(false, Ordering::SeqCst); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> io::Result { + if self.probe_present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(self.id.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> io::Result { if self.panic_in_use { panic!("simulated tier backend in-use panic"); @@ -7685,6 +9273,7 @@ mod tests { struct FakeTierMutationPeer { label: &'static str, calls: Arc>>, + captured_prepares: Option>>>, prepare: std::result::Result, prepare_definitely_rejected: bool, commit: std::result::Result, @@ -7709,6 +9298,7 @@ mod tests { Arc::new(Self { label, calls, + captured_prepares: None, prepare, prepare_definitely_rejected: false, commit, @@ -7716,6 +9306,22 @@ mod tests { }) } + fn boxed_capturing_prepare( + label: &'static str, + calls: Arc>>, + captured_prepares: Arc>>, + ) -> Arc { + Arc::new(Self { + label, + calls, + captured_prepares: Some(captured_prepares), + prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, + commit: Ok(PeerTierMutationState::Committed), + abort: Ok(PeerTierMutationState::Aborted), + }) + } + fn record(&self, call: String) { lock_unpoisoned(&self.calls).push(call); } @@ -7732,6 +9338,10 @@ mod tests { mutation_id: uuid::Uuid, canonical_payload: Bytes, ) -> Result { + if let Some(captured_prepares) = self.captured_prepares.as_ref() { + let intent = TierMutationIntent::decode(mutation_id, &canonical_payload).map_err(Error::other)?; + lock_unpoisoned(captured_prepares).push(intent); + } self.record(format!("{}:prepare:{}:{}", self.label, mutation_id, canonical_payload.len())); match self.prepare { Ok(state) => Ok(state), @@ -7778,6 +9388,16 @@ mod tests { release: Arc, } + struct BlockingPrepareTierMutationPeer { + started: Arc, + release: Arc, + calls: Arc>>, + } + + struct HangingPrepareTierMutationPeer { + calls: Arc>>, + } + #[async_trait::async_trait] impl TierMutationPeer for BlockingCommitTierMutationPeer { fn peer_label(&self) -> String { @@ -7815,6 +9435,79 @@ mod tests { } } + #[async_trait::async_trait] + impl TierMutationPeer for BlockingPrepareTierMutationPeer { + fn peer_label(&self) -> String { + "blocking-prepare-peer".to_string() + } + + async fn prepare_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_payload: Bytes, + ) -> Result { + lock_unpoisoned(&self.calls).push("prepare".to_string()); + self.started.notify_one(); + self.release + .acquire() + .await + .expect("blocking prepare test semaphore should stay open") + .forget(); + Ok(PeerTierMutationState::Prepared) + } + + async fn commit_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_payload: Bytes, + ) -> Result { + lock_unpoisoned(&self.calls).push("commit".to_string()); + Ok(PeerTierMutationState::Committed) + } + + async fn abort_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_prepare_payload: Bytes, + ) -> Result { + lock_unpoisoned(&self.calls).push("abort".to_string()); + Ok(PeerTierMutationState::Aborted) + } + } + + #[async_trait::async_trait] + impl TierMutationPeer for HangingPrepareTierMutationPeer { + fn peer_label(&self) -> String { + "hanging-peer".to_string() + } + + async fn prepare_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_payload: Bytes, + ) -> Result { + lock_unpoisoned(&self.calls).push("hanging-peer:prepare".to_string()); + std::future::pending().await + } + + async fn commit_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_payload: Bytes, + ) -> Result { + Ok(PeerTierMutationState::Committed) + } + + async fn abort_tier_mutation( + &self, + _mutation_id: uuid::Uuid, + _canonical_prepare_payload: Bytes, + ) -> Result { + lock_unpoisoned(&self.calls).push("hanging-peer:abort".to_string()); + Ok(PeerTierMutationState::Aborted) + } + } + impl ConcurrencyTrackingTierMutationPeer { fn boxed( label: &'static str, @@ -7874,9 +9567,10 @@ mod tests { } #[tokio::test] - async fn prepare_tier_mutation_peers_serializes_peer_prepare_writes() { + async fn prepare_tier_mutation_peers_uses_bounded_parallel_fanout() { let mutation_id = uuid::Uuid::from_u128(34); - let intent = prepared_remove_intent("COLD-A", mutation_id); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); let calls = Arc::new(Mutex::new(Vec::new())); let active = Arc::new(AtomicUsize::new(0)); let max_active = Arc::new(AtomicUsize::new(0)); @@ -7887,28 +9581,81 @@ mod tests { ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-d", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-e", calls.clone(), active.clone(), max_active.clone()), ], &intent, ) .await .unwrap_or_else(|failure| panic!("successful prepare fanout should prepare every peer: {}", failure.error.message)); - assert_eq!(prepared.len(), 3); + assert_eq!(prepared.len(), 5); assert_eq!( max_active.load(Ordering::SeqCst), - 1, - "peer prepare fanout must not write the same intent concurrently" + TIER_MUTATION_PEER_FANOUT_CONCURRENCY, + "peer prepare fanout should run independently up to its concurrency bound" ); + let mut calls = lock_unpoisoned(&calls).clone(); + calls.sort(); assert_eq!( - lock_unpoisoned(&calls).as_slice(), - &["peer-a:prepare", "peer-b:prepare", "peer-c:prepare"] + calls, + vec![ + "peer-a:prepare".to_string(), + "peer-b:prepare".to_string(), + "peer-c:prepare".to_string(), + "peer-d:prepare".to_string(), + "peer-e:prepare".to_string(), + ] ); } + #[tokio::test] + async fn prepare_fanout_refills_a_slot_while_an_earlier_peer_is_slow() { + let mutation_id = uuid::Uuid::from_u128(0x38); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); + let calls = Arc::new(Mutex::new(Vec::new())); + let slow_started = Arc::new(Notify::new()); + let slow_release = Arc::new(tokio::sync::Semaphore::new(0)); + let slow: Arc = Arc::new(BlockingPrepareTierMutationPeer { + started: slow_started.clone(), + release: slow_release.clone(), + calls: calls.clone(), + }); + let peers = vec![ + slow, + FakeTierMutationPeer::boxed("peer-b", calls.clone(), Ok(PeerTierMutationState::Prepared)), + FakeTierMutationPeer::boxed("peer-c", calls.clone(), Ok(PeerTierMutationState::Prepared)), + FakeTierMutationPeer::boxed("peer-d", calls.clone(), Ok(PeerTierMutationState::Prepared)), + FakeTierMutationPeer::boxed("peer-e", calls.clone(), Ok(PeerTierMutationState::Prepared)), + ]; + let prepare = tokio::spawn(async move { prepare_tier_mutation_peers(mutation_id, peers, &intent).await }); + slow_started.notified().await; + + tokio::time::timeout(Duration::from_millis(100), async { + loop { + if lock_unpoisoned(&calls).iter().any(|call| call.starts_with("peer-e:prepare:")) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the fifth peer should refill a completed slot without waiting for the first peer"); + slow_release.add_permits(1); + + let prepared = prepare + .await + .expect("work-conserving Prepare task should join") + .unwrap_or_else(|failure| panic!("work-conserving Prepare should succeed: {}", failure.error.message)); + assert_eq!(prepared.len(), 5); + } + #[tokio::test] async fn prepare_tier_mutation_peers_accepts_replayed_committed_peer() { let mutation_id = uuid::Uuid::from_u128(36); - let intent = prepared_remove_intent("COLD-A", mutation_id); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); let calls = Arc::new(Mutex::new(Vec::new())); let peers = vec![ FakeTierMutationPeer::boxed_with_prepare_commit( @@ -7933,15 +9680,75 @@ mod tests { assert_eq!(prepared.len(), 1); let calls = lock_unpoisoned(&calls); assert_eq!(calls.len(), 2); - assert!( - calls[0].starts_with("peer-a:prepare:"), - "replayed peer must be prepared before the remaining peer" + assert!(calls.iter().any(|call| call.starts_with("peer-a:prepare:"))); + assert!(calls.iter().any(|call| call == "peer-b:prepare")); + } + + #[tokio::test(start_paused = true)] + async fn prepare_timeout_is_aggregated_and_compensated_as_ambiguous() { + let mutation_id = uuid::Uuid::from_u128(37); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); + let calls = Arc::new(Mutex::new(Vec::new())); + let definitely_rejected = Arc::new(FakeTierMutationPeer { + label: "rejected-peer", + calls: calls.clone(), + captured_prepares: None, + prepare: Err("definitely rejected"), + prepare_definitely_rejected: true, + commit: Ok(PeerTierMutationState::Committed), + abort: Ok(PeerTierMutationState::Aborted), + }) as Arc; + let hanging = Arc::new(HangingPrepareTierMutationPeer { calls: calls.clone() }) as Arc; + + let failure = match prepare_tier_mutation_peers(mutation_id, vec![hanging, definitely_rejected], &intent).await { + Ok(_) => panic!("a timed-out and rejected prepare fanout must fail"), + Err(failure) => failure, + }; + + assert!(failure.error.message.contains("hanging-peer: request timed out")); + assert!(failure.error.message.contains("rejected-peer")); + assert_eq!(failure.prepared_peers.len(), 1, "only the ambiguous timeout should require abort"); + abort_tier_mutation_peers(&intent, failure.prepared_peers) + .await + .expect("the ambiguous prepare timeout should receive a compensating abort"); + let calls = lock_unpoisoned(&calls); + assert!(calls.iter().any(|call| call == "hanging-peer:prepare")); + assert!(calls.iter().any(|call| call == "hanging-peer:abort")); + assert!(calls.iter().any(|call| call.starts_with("rejected-peer:prepare:"))); + } + + #[tokio::test(start_paused = true)] + async fn prepare_fanout_wide_deadline_does_not_outlive_intent_expiry() { + let mutation_id = uuid::Uuid::from_u128(0x39); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); + intent.expires_at_unix_nanos = i64::try_from(now.saturating_add(1_000_000_000)).unwrap_or(i64::MAX); + let calls = Arc::new(Mutex::new(Vec::new())); + let peers = (0..5) + .map(|_| Arc::new(HangingPrepareTierMutationPeer { calls: calls.clone() }) as Arc) + .collect(); + let started_at = Instant::now(); + + let failure = match prepare_tier_mutation_peers(mutation_id, peers, &intent).await { + Ok(_) => panic!("fanout that reaches intent expiry must fail closed"), + Err(failure) => failure, + }; + + assert!(Instant::now().duration_since(started_at) <= Duration::from_secs(1)); + assert_eq!(failure.prepared_peers.len(), TIER_MUTATION_PEER_FANOUT_CONCURRENCY); + assert!(failure.error.message.contains("fanout deadline expired before request start")); + assert_eq!( + lock_unpoisoned(&calls) + .iter() + .filter(|call| call.as_str() == "hanging-peer:prepare") + .count(), + TIER_MUTATION_PEER_FANOUT_CONCURRENCY ); - assert_eq!(calls[1], "peer-b:prepare"); } #[tokio::test] - async fn commit_tier_mutation_peers_serializes_shared_record_writes() { + async fn commit_tier_mutation_peers_uses_bounded_parallel_fanout() { let mutation_id = uuid::Uuid::from_u128(35); let calls = Arc::new(Mutex::new(Vec::new())); let active = Arc::new(AtomicUsize::new(0)); @@ -7953,6 +9760,8 @@ mod tests { ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-d", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-e", calls.clone(), active.clone(), max_active.clone()), ], "etag-new", ) @@ -7961,14 +9770,54 @@ mod tests { assert_eq!( max_active.load(Ordering::SeqCst), - 1, - "peer commit fanout must not write the same intent concurrently" + TIER_MUTATION_PEER_FANOUT_CONCURRENCY, + "peer commit fanout should run independently up to its concurrency bound" + ); + let mut calls = lock_unpoisoned(&calls).clone(); + calls.sort(); + assert_eq!( + calls, + vec![ + "peer-a:commit".to_string(), + "peer-b:commit".to_string(), + "peer-c:commit".to_string(), + "peer-d:commit".to_string(), + "peer-e:commit".to_string(), + ] ); - assert_eq!(lock_unpoisoned(&calls).as_slice(), &["peer-a:commit", "peer-b:commit", "peer-c:commit"]); } #[tokio::test] - async fn abort_tier_mutation_peers_serializes_shared_record_writes() { + async fn commit_tier_mutation_peers_collects_every_failure() { + let mutation_id = uuid::Uuid::from_u128(38); + let calls = Arc::new(Mutex::new(Vec::new())); + let peers = [("peer-a", "commit-a-failed"), ("peer-b", "commit-b-failed")] + .into_iter() + .map(|(label, message)| { + Arc::new(FakeTierMutationPeer { + label, + calls: calls.clone(), + captured_prepares: None, + prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, + commit: Err(message), + abort: Ok(PeerTierMutationState::Aborted), + }) as Arc + }) + .collect(); + + let err = commit_tier_mutation_peers(mutation_id, peers, "etag-new") + .await + .expect_err("all peer commit failures must be reported"); + + let rendered = err.to_string(); + assert!(rendered.contains("commit-a-failed"), "{rendered}"); + assert!(rendered.contains("commit-b-failed"), "{rendered}"); + assert_eq!(lock_unpoisoned(&calls).len(), 2); + } + + #[tokio::test] + async fn abort_tier_mutation_peers_uses_bounded_parallel_fanout() { let mutation_id = uuid::Uuid::from_u128(36); let intent = prepared_remove_intent("COLD-A", mutation_id); let calls = Arc::new(Mutex::new(Vec::new())); @@ -7981,6 +9830,8 @@ mod tests { ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()), ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-d", calls.clone(), active.clone(), max_active.clone()), + ConcurrencyTrackingTierMutationPeer::boxed("peer-e", calls.clone(), active.clone(), max_active.clone()), ], ) .await @@ -7988,10 +9839,51 @@ mod tests { assert_eq!( max_active.load(Ordering::SeqCst), - 1, - "peer abort fanout must not write the same intent concurrently" + TIER_MUTATION_PEER_FANOUT_CONCURRENCY, + "peer abort fanout should run independently up to its concurrency bound" ); - assert_eq!(lock_unpoisoned(&calls).as_slice(), &["peer-a:abort", "peer-b:abort", "peer-c:abort"]); + let mut calls = lock_unpoisoned(&calls).clone(); + calls.sort(); + assert_eq!( + calls, + vec![ + "peer-a:abort".to_string(), + "peer-b:abort".to_string(), + "peer-c:abort".to_string(), + "peer-d:abort".to_string(), + "peer-e:abort".to_string(), + ] + ); + } + + #[tokio::test] + async fn abort_tier_mutation_peers_collects_every_failure() { + let mutation_id = uuid::Uuid::from_u128(39); + let intent = prepared_remove_intent("COLD-A", mutation_id); + let calls = Arc::new(Mutex::new(Vec::new())); + let peers = [("peer-a", "abort-a-failed"), ("peer-b", "abort-b-failed")] + .into_iter() + .map(|(label, message)| { + Arc::new(FakeTierMutationPeer { + label, + calls: calls.clone(), + captured_prepares: None, + prepare: Ok(PeerTierMutationState::Prepared), + prepare_definitely_rejected: false, + commit: Ok(PeerTierMutationState::Committed), + abort: Err(message), + }) as Arc + }) + .collect(); + + let err = abort_tier_mutation_peers(&intent, peers) + .await + .expect_err("all peer abort failures must be reported"); + + let rendered = err.to_string(); + assert!(rendered.contains("abort-a-failed"), "{rendered}"); + assert!(rendered.contains("abort-b-failed"), "{rendered}"); + assert_eq!(lock_unpoisoned(&calls).len(), 2); } #[tokio::test] @@ -8379,6 +10271,7 @@ mod tests { let old_peer = Arc::new(FakeTierMutationPeer { label: "peer-v3", calls: calls.clone(), + captured_prepares: None, prepare: Err("unsupported tier mutation peer protocol version: 4"), prepare_definitely_rejected: true, commit: Ok(PeerTierMutationState::Committed), @@ -8449,6 +10342,7 @@ mod tests { Arc::new(FakeTierMutationPeer { label: "peer-a", calls: calls.clone(), + captured_prepares: None, prepare: Ok(PeerTierMutationState::Prepared), prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), @@ -8483,11 +10377,14 @@ mod tests { .expect("coordinator intent scan should retain abort recovery evidence"); assert_eq!(intents.len(), 1); assert_eq!(intents[0].state, TierMutationIntentState::Prepared); + let prepared_intent = intents[0].clone(); let mutation_id = intents[0].mutation_id; TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") .await .expect("ambiguous peer cleanup must not leave the coordinator data path fenced"); + let calls_before_unexpired_reload = lock_unpoisoned(&calls).clone(); + TIER_MUTATION_TEST_PEERS .scope( vec![ @@ -8497,7 +10394,32 @@ mod tests { async { TierConfigMgr::reload_handle_with(&manager, store.clone()) .await - .expect("reload should retry the durable coordinator abort"); + .expect("reload should retain an unexpired prepared coordinator intent"); + }, + ) + .await; + assert_eq!( + *lock_unpoisoned(&calls), + calls_before_unexpired_reload, + "recovery must not abort an unexpired prepared owner" + ); + let intents = TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("unexpired coordinator intent should remain readable"); + assert_eq!(intents.len(), 1); + assert_eq!(intents[0].state, TierMutationIntentState::Prepared); + + store.expire_coordinator_intent(prepared_intent).await; + TIER_MUTATION_TEST_PEERS + .scope( + vec![ + FakeTierMutationPeer::boxed("peer-a", calls.clone(), Ok(PeerTierMutationState::Committed)), + FakeTierMutationPeer::boxed("peer-b", calls.clone(), Ok(PeerTierMutationState::Committed)), + ], + async { + TierConfigMgr::reload_handle_with(&manager, store.clone()) + .await + .expect("reload should retry the expired durable coordinator abort"); }, ) .await; @@ -8961,41 +10883,44 @@ mod tests { let update = TierConfigMgr::admin_update_lock(&manager).await; let calls = Arc::new(Mutex::new(Vec::new())); - TIER_MUTATION_TEST_PEERS + TIER_DRIVER_TEST_FACTORY .scope( - vec![ - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-a", - calls.clone(), - Ok(PeerTierMutationState::Committed), - Ok(PeerTierMutationState::Committed), - ), - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-b", - calls.clone(), - Ok(PeerTierMutationState::Prepared), - Ok(PeerTierMutationState::Committed), - ), - FakeTierMutationPeer::boxed_with_prepare_commit( - "peer-c", - calls.clone(), - Ok(PeerTierMutationState::Prepared), - Ok(PeerTierMutationState::Committed), - ), - ], - async { - TierConfigMgr::update_candidate_owned( - &manager, - store.clone(), - candidate, - version, - TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), - update, - None, - ) - .await - .expect("four-hot AddTier reporter replay should publish after prepared peers commit"); - }, + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + vec![ + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-a", + calls.clone(), + Ok(PeerTierMutationState::Committed), + Ok(PeerTierMutationState::Committed), + ), + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-b", + calls.clone(), + Ok(PeerTierMutationState::Prepared), + Ok(PeerTierMutationState::Committed), + ), + FakeTierMutationPeer::boxed_with_prepare_commit( + "peer-c", + calls.clone(), + Ok(PeerTierMutationState::Prepared), + Ok(PeerTierMutationState::Committed), + ), + ], + async { + TierConfigMgr::update_candidate_owned( + &manager, + store.clone(), + candidate, + version, + TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), + update, + None, + ) + .await + .expect("four-hot AddTier reporter replay should publish after prepared peers commit"); + }, + ), ) .await; @@ -10629,6 +12554,52 @@ mod tests { ); } + #[tokio::test] + async fn coordinator_recovery_preserves_unexpired_prepared_owner_on_old_config() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("active Prepared recovery fixture should persist"); + let (_, old_config_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("active Prepared old config should load"); + let mutation_id = uuid::Uuid::from_u128(0x2210); + let mut intent = prepared_remove_intent("COLD-A", mutation_id); + intent.old_config_etag = old_config_etag; + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); + save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent) + .await + .expect("active Prepared coordinator intent should persist"); + let calls = Arc::new(Mutex::new(Vec::new())); + let mut intents = vec![intent]; + + TIER_MUTATION_TEST_PEERS + .scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + calls.clone(), + Ok(PeerTierMutationState::Aborted), + )], + async { + TierConfigMgr::recover_prepared_coordinator_mutation_intents(store.clone(), &mut intents) + .await + .expect("active Prepared recovery should retain its owner fence"); + }, + ) + .await; + + assert_eq!(intents[0].state, TierMutationIntentState::Prepared); + assert!(lock_unpoisoned(&calls).is_empty(), "unexpired owner must not receive recovery Abort"); + let persisted = TierConfigMgr::load_coordinator_mutation_intents(store) + .await + .expect("active Prepared coordinator scan should succeed"); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].state, TierMutationIntentState::Prepared); + } + #[tokio::test] async fn coordinator_mutation_recovery_aborts_expired_unmatched_intent() { let store = Arc::new(CasConfigStore::default()); @@ -10655,6 +12626,7 @@ mod tests { vec![Arc::new(FakeTierMutationPeer { label: "peer-a", calls: calls.clone(), + captured_prepares: None, prepare: Ok(PeerTierMutationState::Prepared), prepare_definitely_rejected: false, commit: Ok(PeerTierMutationState::Committed), @@ -11487,6 +13459,57 @@ mod tests { verify.await.expect("verify task should join").expect("verify should finish"); } + #[tokio::test] + async fn cancelled_admin_verify_finishes_probe_cleanup_and_releases_its_lease() { + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + { + let mut guard = manager.write().await; + guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + guard + .replace_driver("COLD-A", Box::new(backend.clone())) + .expect("verification backend generation should install"); + } + + let verify_manager = manager.clone(); + let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await }); + get_barrier.wait_until_paused().await; + verify.abort(); + assert!( + verify + .await + .expect_err("the outer verify caller should be cancelled") + .is_cancelled() + ); + assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, "COLD-A").await, 1); + + get_barrier.release(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if backend.object_count().await == 0 && TierConfigMgr::active_operation_lease_count(&manager, "COLD-A").await == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the detached verification must finish cleanup and release its lease"); + + let operations = backend.op_log().await; + assert!(matches!( + operations.as_slice(), + [ + MockWarmOp::Put { .. }, + MockWarmOp::Probe { .. }, + MockWarmOp::Get { .. }, + MockWarmOp::Remove { .. }, + MockWarmOp::Probe { .. } + ] + )); + } + #[tokio::test] async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() { let old = LeaseTestBackend::ready("old"); @@ -12374,6 +14397,24 @@ mod tests { .insert(object, (data, "reference-proof-etag".to_string())); } + async fn expire_coordinator_intent(&self, mut intent: TierMutationIntent) { + use crate::services::tier::tier_mutation_intent::{ + TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, TIER_MUTATION_INTENT_RECORD_PREFIX, + tier_mutation_intent_record_object_name, + }; + + intent.expires_at_unix_nanos = 1; + let peer_object = tier_mutation_intent_record_object_name(intent.mutation_id) + .expect("coordinator intent fixture path should build"); + let coordinator_object = + peer_object.replacen(TIER_MUTATION_INTENT_RECORD_PREFIX, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, 1); + self.insert_config_object( + coordinator_object, + intent.encode().expect("expired coordinator intent fixture should encode"), + ) + .await; + } + async fn rewrite_on_next_if_match(&self, object: String, data: Vec) { self.if_match_race_rewrite.lock().await.push_back((object, data)); } @@ -13021,6 +15062,480 @@ mod tests { } } + #[tokio::test] + async fn edit_and_save_with_omitted_credentials_preserves_persisted_and_runtime_secrets() { + const ACCESS_KEY: &str = "persisted-access-value"; + const SECRET_KEY: &str = "persisted-secret-value"; + + let store = Arc::new(CasConfigStore::default()); + let mut tier = build_rustfs_tier("COLD-R"); + let rustfs = tier.rustfs.as_mut().expect("RustFS payload should exist"); + rustfs.access_key = ACCESS_KEY.to_string(); + rustfs.secret_key = SECRET_KEY.to_string(); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-R".to_string(), tier); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("credential-preservation fixture should persist"); + + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + let observed = Arc::new(Mutex::new(Vec::new())); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend.clone(), observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-R", TierCreds::default()), + ), + ) + .await + .expect("omitted credentials should preserve and publish the current credentials"); + + let reloaded = load_tier_config_for_update(store.clone()) + .await + .expect("edited tier config should reload") + .0; + let persisted_rustfs = reloaded.tiers["COLD-R"] + .rustfs + .as_ref() + .expect("persisted RustFS payload should exist"); + assert_eq!(persisted_rustfs.access_key, ACCESS_KEY); + assert_eq!(persisted_rustfs.secret_key, SECRET_KEY); + + let config_object = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("binary tier config should remain persisted") + .0 + .clone(); + assert!( + !config_object + .windows(TIER_CREDENTIAL_REDACTED.len()) + .any(|window| window == TIER_CREDENTIAL_REDACTED.as_bytes()), + "the API placeholder must never be persisted as a credential" + ); + + let (runtime_access_key, runtime_secret_key) = { + let manager = manager.read().await; + let runtime = registered_tier_driver_runtime(&manager).expect("driver runtime should be registered"); + let runtime = lock_unpoisoned(&runtime); + let generation = runtime + .generations + .get("COLD-R") + .expect("edited generation should be installed"); + let rustfs = generation + .tier_config + .rustfs + .as_ref() + .expect("runtime RustFS payload should exist"); + (rustfs.access_key.clone(), rustfs.secret_key.clone()) + }; + assert_eq!(runtime_access_key, ACCESS_KEY); + assert_eq!(runtime_secret_key, SECRET_KEY); + + let api_view = manager + .read() + .await + .get("COLD-R") + .expect("edited tier should remain visible through the admin view"); + assert_eq!( + api_view.rustfs.expect("admin RustFS payload should exist").secret_key, + TIER_CREDENTIAL_REDACTED + ); + let observed = lock_unpoisoned(&observed); + assert_eq!(observed.len(), 1); + assert_eq!( + observed[0] + .rustfs + .as_ref() + .expect("backend factory should observe the RustFS payload") + .secret_key, + SECRET_KEY + ); + drop(observed); + + let operations = backend.op_log().await; + assert_eq!(operations.len(), 5); + assert!(matches!(&operations[0], MockWarmOp::Put { .. })); + assert!(matches!(&operations[1], MockWarmOp::Probe { .. })); + assert!(matches!(&operations[2], MockWarmOp::Get { .. })); + assert!(matches!(&operations[3], MockWarmOp::Remove { .. })); + assert!(matches!(&operations[4], MockWarmOp::Probe { .. })); + } + + #[tokio::test] + async fn edit_and_save_rotates_credentials_while_an_active_lifecycle_rule_references_the_tier() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("credential rotation fixture should persist"); + store.add_listed_version(ObjectInfo { + bucket: "photos".to_string(), + name: "safe.txt".to_string(), + ..Default::default() + }); + store.add_lifecycle_config( + "photos", + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("move-current".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: Some(vec![Transition { + days: Some(1), + date: None, + storage_class: Some(TransitionStorageClass::from_static("COLD-A")), + }]), + }], + }, + ); + + let manager = TierConfigMgr::new(); + let backend = RecordingWarmBackend::new(); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::edit_and_save_with( + &manager, + store.clone(), + "COLD-A", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + ), + ) + .await + .expect("same-destination credential rotation must not be treated as a lifecycle rebind"); + + let reloaded = load_tier_config_for_update(store) + .await + .expect("rotated tier config should reload") + .0; + let rustfs = reloaded.tiers["COLD-A"] + .rustfs + .as_ref() + .expect("RustFS payload should remain"); + assert_eq!(rustfs.access_key, "rotated-access"); + assert_eq!(rustfs.secret_key, "rotated-secret"); + } + + async fn assert_edit_rejected_before_distributed_commit( + backend: RecordingWarmBackend, + credentials: TierCreds, + expected_code: &str, + expected_backend_builds: usize, + ) { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("rejected-edit fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("old")); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should be available") + .generation(); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::new(Mutex::new(Vec::new())); + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-A", credentials), + ), + ) + .await + .expect_err("credential validation must fail before distributed prepare"); + + let TierConfigUpdateError::Mutation(err) = err else { + panic!("credential validation should be reported as a mutation error: {err:?}"); + }; + assert_eq!(err.code, expected_code); + assert!(lock_unpoisoned(&peer_calls).is_empty()); + assert_eq!(lock_unpoisoned(&observed).len(), expected_backend_builds); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + let current = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("the old generation should be restored after validation failure"); + assert_eq!(current.generation(), old_generation); + assert_eq!( + current + .inner + .tier_config + .rustfs + .as_ref() + .expect("runtime RustFS payload should remain") + .secret_key, + "sk" + ); + } + + #[tokio::test] + async fn edit_and_save_rejects_half_credentials_before_peer_prepare_or_config_cas() { + assert_edit_rejected_before_distributed_commit( + RecordingWarmBackend::new(), + TierCreds { + access_key: "rotated-access".to_string(), + ..Default::default() + }, + &ERR_TIER_MISSING_CREDENTIALS.code, + 0, + ) + .await; + } + + #[tokio::test] + async fn edit_and_save_rejects_canonical_azure_service_principal_before_peer_prepare() { + let credentials: TierCreds = serde_json::from_value(serde_json::json!({ + "azSP": { + "TenantID": "tenant", + "ClientID": "client", + "ClientSecret": "service-principal-secret" + } + })) + .expect("canonical madmin credentials should decode"); + assert_edit_rejected_before_distributed_commit( + RecordingWarmBackend::new(), + credentials, + &ERR_TIER_INVALID_CONFIG.code, + 0, + ) + .await; + } + + #[tokio::test] + async fn edit_and_save_rejects_legacy_azure_options_before_probe_or_peer_prepare() { + let store = Arc::new(CasConfigStore::default()); + let mut legacy = build_azure_tier("account-a"); + legacy.azure.as_mut().expect("Azure payload should exist").storage_class = "HOT".to_string(); + let mut persisted = empty_mgr(); + persisted + .tiers + .insert("COLD-AZURE".to_string(), legacy.clone_with_credentials()); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("legacy Azure fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + manager.tiers.insert("COLD-AZURE".to_string(), legacy); + manager + .replace_driver("COLD-AZURE", Box::new(LeaseTestBackend::ready("old"))) + .expect("legacy generation should install"); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-AZURE") + .await + .expect("legacy generation should be available") + .generation(); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::new(Mutex::new(Vec::new())); + + let err = TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(RecordingWarmBackend::new(), observed.clone()), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with(&manager, store.clone(), "COLD-AZURE", TierCreds::default()), + ), + ) + .await + .expect_err("legacy unsupported Azure options must fail closed on edit"); + + let TierConfigUpdateError::Mutation(err) = err else { + panic!("legacy Azure validation should be a mutation error: {err:?}"); + }; + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("storageClass"), "{}", err.message); + assert!(lock_unpoisoned(&observed).is_empty(), "validation must precede backend preparation"); + assert!(lock_unpoisoned(&peer_calls).is_empty(), "validation must precede peer prepare"); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + let current = TierConfigMgr::acquire_operation_lease(&manager, "COLD-AZURE") + .await + .expect("legacy generation should remain available"); + assert_eq!(current.generation(), old_generation); + assert_eq!( + current + .inner + .tier_config + .azure + .as_ref() + .expect("runtime Azure payload should remain") + .storage_class, + "HOT" + ); + } + + #[tokio::test] + async fn edit_and_save_probe_failure_keeps_old_generation_and_skips_peer_prepare() { + let backend = RecordingWarmBackend::new(); + backend.set_reject_credentials(true).await; + assert_edit_rejected_before_distributed_commit( + backend, + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + &ERR_TIER_PERM_ERR.code, + 1, + ) + .await; + } + + #[tokio::test(start_paused = true)] + async fn edit_and_save_get_timeout_cleans_probe_and_keeps_config_and_generation() { + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("timeout fixture should persist"); + let before = store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should be present") + .clone(); + + let manager = TierConfigMgr::new(); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("old")); + } + let old_generation = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should be available") + .generation(); + let backend = RecordingWarmBackend::new(); + let get_barrier = backend.arm_get_barrier().await; + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let edit = TIER_DRIVER_TEST_FACTORY.scope( + recording_driver_factory(backend.clone(), Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + peer_calls.clone(), + Ok(PeerTierMutationState::Committed), + )], + TierConfigMgr::edit_and_save_with( + &manager, + store.clone(), + "COLD-A", + TierCreds { + access_key: "rotated-access".to_string(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }, + ), + ), + ); + tokio::pin!(edit); + tokio::select! { + _ = get_barrier.wait_until_paused() => {} + result = &mut edit => panic!("edit completed before the probe GET timeout: {result:?}"), + } + tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await; + let err = edit.await.expect_err("a hanging probe GET must time out"); + let TierConfigUpdateError::Mutation(err) = err else { + panic!("probe timeout should be reported as a mutation error: {err:?}"); + }; + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(lock_unpoisoned(&peer_calls).is_empty()); + assert_eq!( + store + .objects + .lock() + .await + .get(&tier_config_path(TIER_CONFIG_FILE)) + .expect("base config should remain present"), + &before + ); + assert_eq!( + TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should remain available") + .generation(), + old_generation + ); + let operations = backend.op_log().await; + assert!(matches!(operations.first(), Some(MockWarmOp::Put { .. }))); + assert!( + operations + .iter() + .any(|operation| matches!(operation, MockWarmOp::Remove { .. })) + ); + assert!(matches!(operations.last(), Some(MockWarmOp::Probe { .. }))); + } + #[tokio::test] async fn remove_and_clear_full_update_paths_preserve_force() { let remove_store = Arc::new(CasConfigStore::default()); @@ -13690,13 +16205,17 @@ mod tests { .await .expect("add fixture should persist"); - TierConfigMgr::update_candidate_with_config_lock( - &manager, - store.clone(), - TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true), - ) - .await - .expect("add proof must ignore unchanged durable tiers missing from the stale manager"); + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TierConfigMgr::update_candidate_with_config_lock( + &manager, + store.clone(), + TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true), + ), + ) + .await + .expect("add proof must ignore unchanged durable tiers missing from the stale manager"); let reloaded = load_tier_config_for_update(store) .await @@ -13706,6 +16225,84 @@ mod tests { assert!(reloaded.tiers.contains_key("COLD-B")); } + #[tokio::test] + async fn nested_only_legacy_add_builds_nonempty_coordinator_target_and_fans_out() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("empty tier config fixture should persist"); + let legacy_wire: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "s3", + "s3": { + "name": "COLD-LEGACY", + "endpoint": "https://s3.example.invalid", + "accessKey": "access", + "secretKey": "secret", + "bucket": "archive", + "prefix": "objects", + "region": "us-east-1", + "storageClass": "STANDARD" + } + })) + .expect("legacy RustFS nested-name AddTier payload should decode"); + assert!(legacy_wire.name.is_empty()); + let mutation = TierCandidateMutation::add(legacy_wire, true) + .expect("legacy nested name must normalize before constructing the Add mutation"); + assert_eq!(mutation.explicit_tier_name(), Some("COLD-LEGACY")); + + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let prepared_intents = Arc::new(Mutex::new(Vec::new())); + let backend = RecordingWarmBackend::new(); + TIER_DRIVER_TEST_FACTORY + .scope( + recording_driver_factory(backend, Arc::new(Mutex::new(Vec::new()))), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed_capturing_prepare( + "peer-a", + peer_calls.clone(), + prepared_intents.clone(), + )], + TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation), + ), + ) + .await + .expect("legacy nested-name Add must run the full coordinator fanout"); + + let prepared_intents = lock_unpoisoned(&prepared_intents); + assert_eq!(prepared_intents.len(), 1); + assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add); + assert_eq!(prepared_intents[0].affected_targets.len(), 1); + assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY"); + assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none()); + assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some()); + drop(prepared_intents); + + let peer_calls = lock_unpoisoned(&peer_calls).clone(); + let prepare_index = peer_calls + .iter() + .position(|call| call.starts_with("peer-a:prepare:")) + .expect("legacy Add should prepare the peer"); + let commit_index = peer_calls + .iter() + .position(|call| call.starts_with("peer-a:commit:")) + .expect("legacy Add should commit the peer"); + assert!(prepare_index < commit_index, "{peer_calls:?}"); + + let reloaded = load_tier_config_for_update(store) + .await + .expect("legacy Add result should reload") + .0; + let persisted = reloaded + .tiers + .get("COLD-LEGACY") + .expect("normalized tier name should be persisted"); + assert_eq!(persisted.name, "COLD-LEGACY"); + assert_eq!(persisted.s3.as_ref().expect("persisted S3 payload should exist").name, "COLD-LEGACY"); + assert!(manager.read().await.tiers.contains_key("COLD-LEGACY")); + } + #[tokio::test] async fn config_update_refuses_pending_prepared_recovery_before_advancing_etag() { let manager = TierConfigMgr::new(); @@ -13789,11 +16386,21 @@ mod tests { .await .expect("coordinator intent should remain recoverable"); assert_eq!(intents.len(), 1); - assert_eq!(intents[0].state, TierMutationIntentState::Aborted); + assert_eq!(intents[0].state, TierMutationIntentState::Prepared); TierConfigMgr::reload_handle_with(&manager, store.clone()) .await - .expect("reload should clean the aborted pending recovery"); + .expect("reload should retain the unexpired prepared owner"); + let intents = TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("unexpired coordinator intent should remain recoverable"); + assert_eq!(intents.len(), 1); + assert_eq!(intents[0].state, TierMutationIntentState::Prepared); + + store.expire_coordinator_intent(prepared).await; + TierConfigMgr::reload_handle_with(&manager, store.clone()) + .await + .expect("reload should clean the expired pending recovery"); assert!( TierConfigMgr::load_coordinator_mutation_intents(store) .await @@ -13931,7 +16538,7 @@ mod tests { } #[tokio::test] - async fn config_write_lock_survives_cancelled_wrapped_update_until_detached_publish_finishes() { + async fn config_write_lock_is_released_while_cancelled_update_drains_leases() { let manager = TierConfigMgr::new(); { let mut guard = manager.write().await; @@ -13971,11 +16578,11 @@ mod tests { .new_ns_lock(RUSTFS_META_BUCKET, &config_file) .await .expect("competing tier config lock should be created"); - let err = competing_lock - .get_write_lock(Duration::from_millis(20)) + let competing_guard = competing_lock + .get_write_lock(Duration::from_secs(1)) .await - .expect_err("detached update must keep the config write lock until it finishes"); - assert!(matches!(err, rustfs_lock::LockError::Timeout { .. })); + .expect("durable Prepared must allow the config lock to release while leases drain"); + drop(competing_guard); drop(old); tokio::time::timeout(Duration::from_secs(1), async { @@ -13985,10 +16592,529 @@ mod tests { }) .await .expect("detached update must finish after its operation lease drains"); - competing_lock - .get_write_lock(Duration::from_secs(1)) + } + + #[tokio::test] + #[serial_test::serial] + async fn backend_validation_runs_without_holding_the_config_namespace_lock() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) .await - .expect("tier config lock should release after detached update finishes"); + .expect("backend-validation fixture should persist"); + let (barrier, _barrier_guard) = install_tier_driver_build_barrier("COLD-A"); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), + ), + ) + .await + }); + barrier.arrived.notified().await; + + let competing_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, &tier_config_lock_path()) + .await + .expect("competing config lock should be created"); + let competing_guard = competing_lock + .get_write_lock(Duration::from_millis(100)) + .await + .expect("remote backend validation must not hold the config namespace lock"); + drop(competing_guard); + + barrier.release.add_permits(1); + update + .await + .expect("backend-validation update task should join") + .expect("backend-validation update should succeed"); + } + + #[tokio::test] + #[serial_test::serial] + async fn backend_validation_rejects_a_changed_persisted_etag_before_prepare() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("stale backend-validation fixture should persist"); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let (barrier, _barrier_guard) = install_tier_driver_build_barrier("COLD-A"); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update_peer_calls = peer_calls.clone(); + let update = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + vec![FakeTierMutationPeer::boxed( + "peer-a", + update_peer_calls, + Ok(PeerTierMutationState::Prepared), + )], + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), + ), + ), + ) + .await + }); + barrier.arrived.notified().await; + + let config_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, &tier_config_lock_path()) + .await + .expect("competing config lock should be created"); + let config_guard = config_lock + .get_write_lock(Duration::from_millis(100)) + .await + .expect("backend validation should leave the persisted generation available"); + let (mut competing, competing_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("competing config should load"); + competing.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + competing + .save_tiering_config_if_current(store.clone(), competing_etag.as_deref()) + .await + .expect("competing config generation should commit"); + drop(config_guard); + barrier.release.add_permits(1); + + let err = update + .await + .expect("stale backend-validation update task should join") + .expect_err("a changed persisted ETag must reject the prevalidated candidate"); + let TierConfigUpdateError::Load(err) = err else { + panic!("stale backend validation should report a load conflict"); + }; + assert!( + err.to_string() + .contains("changed while the remote backend mutation was being validated") + ); + let current = load_tier_config_for_update(store.clone()) + .await + .expect("winning config should reload") + .0; + assert!(current.tiers.contains_key("COLD-B")); + assert!(!current.tiers.contains_key("COLD-A")); + assert!(lock_unpoisoned(&peer_calls).is_empty(), "stale validation must not send peer Prepare"); + assert!( + TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("stale validation intent scan should succeed") + .is_empty(), + "stale validation must not leave a coordinator Prepared intent" + ); + } + + #[tokio::test] + async fn detached_backend_validation_panic_is_reported_as_mutation_failure() { + let panic_factory: TierDriverTestFactory = Arc::new(|_| panic!("injected backend factory panic")); + let result = TIER_DRIVER_TEST_FACTORY + .scope( + panic_factory, + TierConfigMgr::prevalidate_candidate_owned( + empty_mgr(), + None, + TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true), + ), + ) + .await; + let err = match result { + Ok(_) => panic!("a detached backend validation panic must be observable"), + Err(err) => err, + }; + + let TierConfigUpdateError::Mutation(err) = err else { + panic!("backend validation panic should retain the mutation error category"); + }; + assert!(err.message.contains("backend validation task failed"), "{}", err.message); + } + + #[tokio::test] + async fn peer_prepare_wait_releases_exclusive_locks_and_stale_candidate_cannot_publish() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("peer-wait lock fixture should persist"); + let started = Arc::new(Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let peer_calls = Arc::new(Mutex::new(Vec::new())); + let peer: Arc = Arc::new(BlockingPrepareTierMutationPeer { + started: started.clone(), + release: release.clone(), + calls: peer_calls.clone(), + }); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + vec![peer], + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ) + .await + }); + started.notified().await; + + let admin_guard = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::admin_update_lock(&manager)) + .await + .expect("peer Prepare wait should release the local admin mutex"); + let competing_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, &tier_config_lock_path()) + .await + .expect("peer-wait competing config lock should be created"); + let competing_guard = competing_lock + .get_write_lock(Duration::from_millis(100)) + .await + .expect("peer Prepare wait should release the namespace lock"); + let (mut competing, competing_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("peer-wait competing config should load"); + competing.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + competing + .save_tiering_config_if_current(store.clone(), competing_etag.as_deref()) + .await + .expect("peer-wait competing config should commit"); + drop(competing_guard); + drop(admin_guard); + release.add_permits(1); + + let err = update + .await + .expect("peer-wait update task should join") + .expect_err("the stale peer-wait candidate must not publish"); + let TierConfigUpdateError::Save(err) = err else { + panic!("stale peer-wait candidate should fail precommit revalidation"); + }; + assert!(err.to_string().contains("configuration changed"), "{err}"); + assert_eq!(lock_unpoisoned(&peer_calls).as_slice(), &["prepare".to_string(), "abort".to_string()]); + let current = load_tier_config_for_update(store) + .await + .expect("winning peer-wait config should reload") + .0; + assert!(current.tiers.contains_key("COLD-A")); + assert!(current.tiers.contains_key("COLD-B")); + } + + #[tokio::test] + async fn peer_commit_wait_releases_exclusive_locks_after_durable_commit() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("peer-commit lock fixture should persist"); + let started = Arc::new(Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let peer: Arc = Arc::new(BlockingCommitTierMutationPeer { + started: started.clone(), + release: release.clone(), + }); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + vec![peer], + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ) + .await + }); + started.notified().await; + + let admin_guard = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::admin_update_lock(&manager)) + .await + .expect("peer Commit wait should release the local admin mutex"); + let competing_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, &tier_config_lock_path()) + .await + .expect("peer-commit competing config lock should be created"); + let competing_guard = competing_lock + .get_write_lock(Duration::from_millis(100)) + .await + .expect("durable coordinator Commit should release the namespace lock before peer Commit"); + let committed = load_tier_config_for_update(store.clone()) + .await + .expect("committed config should be readable during peer Commit") + .0; + assert!(!committed.tiers.contains_key("COLD-A")); + drop(competing_guard); + drop(admin_guard); + release.add_permits(1); + + update + .await + .expect("peer-commit update task should join") + .expect("peer-commit update should publish after the peer responds"); + assert!(!manager.read().await.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + #[serial_test::serial] + async fn reference_proof_releases_exclusive_locks_and_stale_candidate_cannot_publish() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("reference-proof lock fixture should persist"); + let barrier = tier_reference_proof_test_barrier(); + let scoped_barrier = barrier.clone(); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_REFERENCE_PROOF_TEST_BARRIER + .scope( + scoped_barrier, + TIER_DRIVER_TEST_FACTORY.scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ), + ) + .await + }); + barrier.arrived.notified().await; + + let admin_guard = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::admin_update_lock(&manager)) + .await + .expect("reference proof should release the local admin update mutex"); + let competing_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, &tier_config_lock_path()) + .await + .expect("competing config lock should be created"); + let competing_guard = competing_lock + .get_write_lock(Duration::from_millis(100)) + .await + .expect("durable Prepared must allow the namespace lock to release during reference proof"); + drop(admin_guard); + + let (mut competing, competing_etag) = load_tier_config_for_update(store.clone()) + .await + .expect("competing config should load during reference proof"); + competing.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + competing + .save_tiering_config_if_current(store.clone(), competing_etag.as_deref()) + .await + .expect("competing config should commit while the stale proof is paused"); + drop(competing_guard); + + barrier.release.add_permits(1); + let err = update + .await + .expect("reference-proof update task should join") + .expect_err("the stale reference-proof candidate must not publish"); + let TierConfigUpdateError::Save(err) = err else { + panic!("stale reference proof should fail precommit revalidation"); + }; + assert!(err.to_string().contains("configuration changed"), "{err}"); + let current = load_tier_config_for_update(store) + .await + .expect("winning reference-proof config should reload") + .0; + assert!(current.tiers.contains_key("COLD-A")); + assert!(current.tiers.contains_key("COLD-B")); + } + + #[tokio::test] + #[serial_test::serial] + async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("cancel-after-prepare fixture should persist"); + let barrier = tier_reference_proof_test_barrier(); + let scoped_barrier = barrier.clone(); + let update_manager = manager.clone(); + let update_store = store.clone(); + let caller = tokio::spawn(async move { + TIER_REFERENCE_PROOF_TEST_BARRIER + .scope( + scoped_barrier, + TIER_DRIVER_TEST_FACTORY.scope( + healthy_driver_factory(), + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ), + ) + .await + }); + barrier.arrived.notified().await; + assert_eq!( + TierConfigMgr::load_coordinator_mutation_intents(store.clone()) + .await + .expect("durable coordinator intent should remain readable") + .len(), + 1, + "reference proof must start only after the durable Prepared intent exists" + ); + + caller.abort(); + barrier.release.add_permits(1); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let current = load_tier_config_for_update(store.clone()) + .await + .expect("detached mutation config should remain readable") + .0; + if !current.tiers.contains_key("COLD-A") { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the detached Prepared mutation should converge after caller cancellation"); + assert!(!manager.read().await.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + async fn precommit_revalidation_rejects_changed_or_expired_intent_identity() { + let store = Arc::new(CasConfigStore::default()); + let mut config = empty_mgr(); + config.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + config + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("precommit revalidation fixture should persist"); + let (candidate, version) = load_tier_config_for_update(store.clone()) + .await + .expect("precommit revalidation fixture should load"); + let candidate_digest = tier_config_candidate_digest(&candidate).expect("candidate digest should build"); + let mut intent = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(40)); + intent.old_config_etag = version.clone(); + intent.candidate_digest = candidate_digest; + intent.expires_at_unix_nanos = tier_mutation_intent_expiry_unix_nanos(); + save_coordinator_tier_mutation_intent(store.clone(), Some(&intent)) + .await + .expect("prepared coordinator intent should persist"); + + TierConfigMgr::revalidate_prepared_candidate_before_commit( + store.clone(), + version.as_deref(), + candidate_digest, + Some(&intent), + ) + .await + .expect("the exact live Prepared identity should revalidate"); + + let stale_version = "stale-config-etag".to_string(); + let err = TierConfigMgr::revalidate_prepared_candidate_before_commit( + store.clone(), + Some(&stale_version), + candidate_digest, + Some(&intent), + ) + .await + .expect_err("an ETag mismatch must block config CAS"); + assert!(err.to_string().contains("configuration changed"), "{err}"); + + let err = TierConfigMgr::revalidate_prepared_candidate_before_commit( + store.clone(), + version.as_deref(), + [0xEE; 32], + Some(&intent), + ) + .await + .expect_err("a candidate digest mismatch must block config CAS"); + assert!(err.to_string().contains("candidate digest changed"), "{err}"); + + let mut mismatched_identity = intent.clone(); + mismatched_identity.affected_targets[0].old_backend_identity = Some([0xAB; 32]); + let err = TierConfigMgr::revalidate_prepared_candidate_before_commit( + store.clone(), + version.as_deref(), + candidate_digest, + Some(&mismatched_identity), + ) + .await + .expect_err("a changed Prepared target identity must block config CAS"); + assert!(err.to_string().contains("intent changed"), "{err}"); + + advance_tier_coordinator_mutation_intent_record_idempotent( + store.clone(), + intent.mutation_id, + TierMutationIntentState::Aborted, + None, + ) + .await + .expect("fixture intent should advance to Aborted"); + let err = TierConfigMgr::revalidate_prepared_candidate_before_commit( + store.clone(), + version.as_deref(), + candidate_digest, + Some(&intent), + ) + .await + .expect_err("an Aborted durable identity must block config CAS"); + assert!(err.to_string().contains("intent changed"), "{err}"); + + let mut expired = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(41)); + expired.old_config_etag = version.clone(); + expired.candidate_digest = candidate_digest; + expired.expires_at_unix_nanos = 1; + save_coordinator_tier_mutation_intent(store.clone(), Some(&expired)) + .await + .expect("expired Prepared fixture should persist"); + let err = TierConfigMgr::revalidate_prepared_candidate_before_commit( + store, + version.as_deref(), + candidate_digest, + Some(&expired), + ) + .await + .expect_err("an expired Prepared identity must block config CAS"); + assert!(err.to_string().contains("expired before config commit"), "{err}"); } #[tokio::test] diff --git a/crates/ecstore/src/services/tier/tier_admin.rs b/crates/ecstore/src/services/tier/tier_admin.rs index 1b3519c02..32cd844a0 100644 --- a/crates/ecstore/src/services/tier/tier_admin.rs +++ b/crates/ecstore/src/services/tier/tier_admin.rs @@ -18,25 +18,203 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; -#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Clone)] +#[serde(default)] +pub struct TierServicePrincipalAuth { + #[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")] + pub tenant_id: String, + #[serde(rename = "ClientID", alias = "clientID", alias = "client_id")] + pub client_id: String, + #[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")] + pub client_secret: String, +} + +impl TierServicePrincipalAuth { + pub(crate) fn is_empty(&self) -> bool { + self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty() + } +} + +impl std::fmt::Debug for TierServicePrincipalAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TierServicePrincipalAuth") + .field("tenant_id", &self.tenant_id) + .field("client_id", &self.client_id) + .field("client_secret", &"REDACTED") + .finish() + } +} + +#[derive(Serialize, Deserialize, Default, Clone)] #[serde(default)] pub struct TierCreds { - #[serde(rename = "accessKey")] + #[serde(rename = "access", alias = "accessKey")] pub access_key: String, - #[serde(rename = "secretKey")] + #[serde(rename = "secret", alias = "secretKey")] pub secret_key: String, - #[serde(rename = "awsRole")] + #[serde(rename = "awsrole", alias = "awsRole")] pub aws_role: bool, - #[serde(rename = "awsRoleWebIdentityTokenFile")] + #[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")] pub aws_role_web_identity_token_file: String, - #[serde(rename = "awsRoleArn")] + #[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")] pub aws_role_arn: String, - //azsp: ServicePrincipalAuth, + #[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")] + pub azure_service_principal: TierServicePrincipalAuth, - //#[serde(rename = "credsJson")] + #[serde( + rename = "creds", + alias = "credsJson", + alias = "credsJSON", + alias = "creds_json", + default, + skip_serializing_if = "Vec::is_empty", + with = "base64_bytes" + )] pub creds_json: Vec, } + +impl std::fmt::Debug for TierCreds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TierCreds") + .field("access_key", &self.access_key) + .field("secret_key", &"REDACTED") + .field("aws_role", &self.aws_role) + .field( + "aws_role_web_identity_token_file", + &(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"), + ) + .field("aws_role_arn", &self.aws_role_arn) + .field("azure_service_principal", &self.azure_service_principal) + .field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED")) + .finish() + } +} + +mod base64_bytes { + use super::*; + + pub(super) fn serialize(value: &[u8], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value)) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum EncodedBytes { + Base64(String), + Legacy(Vec), + } + + match EncodedBytes::deserialize(deserializer)? { + EncodedBytes::Base64(value) => base64_simd::STANDARD + .decode_to_vec(value.as_bytes()) + .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes())) + .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes())) + .map_err(de::Error::custom), + EncodedBytes::Legacy(value) => Ok(value), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() { + let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes(); + let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ=="; + let creds: TierCreds = serde_json::from_value(serde_json::json!({ + "access": "access", + "secret": "secret", + "awsrole": false, + "creds": encoded, + })) + .expect("madmin tier credentials should decode"); + + assert_eq!(creds.access_key, "access"); + assert_eq!(creds.secret_key, "secret"); + assert_eq!(creds.creds_json.as_slice(), &service_account[..]); + + let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode"); + assert_eq!(wire["access"], "access"); + assert_eq!(wire["secret"], "secret"); + assert_eq!(wire["creds"], encoded); + assert!(wire.get("accessKey").is_none()); + assert!(wire.get("secretKey").is_none()); + + let legacy: TierCreds = serde_json::from_value(serde_json::json!({ + "accessKey": "legacy-access", + "secretKey": "legacy-secret", + "credsJson": service_account, + })) + .expect("the former RustFS field names and byte-array encoding should remain readable"); + assert_eq!(legacy.access_key, "legacy-access"); + assert_eq!(legacy.secret_key, "legacy-secret"); + assert_eq!(legacy.creds_json.as_slice(), &service_account[..]); + } + + #[test] + fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() { + let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes(); + for encoder in [ + base64_simd::STANDARD, + base64_simd::STANDARD_NO_PAD, + base64_simd::URL_SAFE, + base64_simd::URL_SAFE_NO_PAD, + ] { + let encoded = encoder.encode_to_string(service_account); + let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded })) + .expect("all supported madmin base64 forms should decode"); + assert_eq!(creds.creds_json, service_account); + } + } + + #[test] + fn tier_creds_debug_redacts_secret_payloads() { + let creds = TierCreds { + access_key: "access".to_string(), + secret_key: "tier-secret-value".to_string(), + aws_role_web_identity_token_file: "/var/run/private-token".to_string(), + creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(), + ..Default::default() + }; + + let rendered = format!("{creds:?}"); + assert!(!rendered.contains("tier-secret-value")); + assert!(!rendered.contains("/var/run/private-token")); + assert!(!rendered.contains("gcs-private-key-value")); + } + + #[test] + fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() { + let creds: TierCreds = serde_json::from_value(serde_json::json!({ + "azSP": { + "TenantID": "tenant", + "ClientID": "client", + "ClientSecret": "service-principal-secret" + } + })) + .expect("canonical madmin azure service principal credentials should decode"); + + assert_eq!(creds.azure_service_principal.tenant_id, "tenant"); + assert_eq!(creds.azure_service_principal.client_id, "client"); + assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret"); + let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode"); + assert_eq!(wire["azSP"]["TenantID"], "tenant"); + assert_eq!(wire["azSP"]["ClientID"], "client"); + assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret"); + assert!(!format!("{creds:?}").contains("service-principal-secret")); + } +} diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index 94fe4e408..3d4833727 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[ pub enum TierType { #[default] Unsupported, - #[serde(rename = "s3")] + #[serde(rename = "s3", alias = "S3")] S3, #[serde(rename = "wasabi")] Wasabi, @@ -58,7 +58,7 @@ pub enum TierType { Huaweicloud, #[serde(rename = "azure")] Azure, - #[serde(rename = "gcs")] + #[serde(rename = "gcs", alias = "GCS")] GCS, #[serde(rename = "r2")] R2, @@ -138,16 +138,18 @@ impl TierType { } } -#[derive(Default, Debug, Serialize, Deserialize)] +pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED"; + +#[derive(Default, Serialize, Deserialize)] #[serde(default)] pub struct TierConfig { #[serde(skip)] pub version: String, - #[serde(rename = "type")] + #[serde(rename = "type", alias = "Type")] pub tier_type: TierType, - #[serde(skip)] + #[serde(rename = "Name", alias = "name", skip_serializing)] pub name: String, - #[serde(rename = "s3", skip_serializing_if = "Option::is_none")] + #[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")] pub s3: Option, #[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")] pub wasabi: Option, @@ -159,7 +161,7 @@ pub struct TierConfig { pub huaweicloud: Option, #[serde(rename = "azure", skip_serializing_if = "Option::is_none")] pub azure: Option, - #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")] + #[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")] pub gcs: Option, #[serde(rename = "r2", skip_serializing_if = "Option::is_none")] pub r2: Option, @@ -170,109 +172,91 @@ pub struct TierConfig { } impl Clone for TierConfig { - fn clone(&self) -> TierConfig { - let mut s3 = None; - let mut wasabi = None; - let mut r = None; - let mut compatible_backend = None; - let mut aliyun = None; - let mut tencent = None; - let mut huaweicloud = None; - let mut azure = None; - let mut gcs = None; - let mut r2 = None; - match self.tier_type { - TierType::S3 => { - if let Some(s3_) = self.s3.as_ref() { - let mut s3_clone = s3_.clone(); - s3_clone.secret_key = "REDACTED".to_string(); - s3 = Some(s3_clone); - } - } - TierType::Wasabi => { - if let Some(wasabi_) = self.wasabi.as_ref() { - let mut wasabi_clone = wasabi_.clone(); - wasabi_clone.secret_key = "REDACTED".to_string(); - wasabi = Some(wasabi_clone); - } - } - TierType::RustFS => { - if let Some(r_) = self.rustfs.as_ref() { - let mut r_clone = r_.clone(); - r_clone.secret_key = "REDACTED".to_string(); - r = Some(r_clone); - } - } - TierType::MinIO => { - if let Some(compatible_backend_) = self.minio.as_ref() { - let mut compatible_backend_clone = compatible_backend_.clone(); - compatible_backend_clone.secret_key = "REDACTED".to_string(); - compatible_backend = Some(compatible_backend_clone); - } - } - TierType::Aliyun => { - if let Some(aliyun_) = self.aliyun.as_ref() { - let mut aliyun_clone = aliyun_.clone(); - aliyun_clone.secret_key = "REDACTED".to_string(); - aliyun = Some(aliyun_clone); - } - } - TierType::Tencent => { - if let Some(tencent_) = self.tencent.as_ref() { - let mut tencent_clone = tencent_.clone(); - tencent_clone.secret_key = "REDACTED".to_string(); - tencent = Some(tencent_clone); - } - } - TierType::Huaweicloud => { - if let Some(huaweicloud_) = self.huaweicloud.as_ref() { - let mut huaweicloud_clone = huaweicloud_.clone(); - huaweicloud_clone.secret_key = "REDACTED".to_string(); - huaweicloud = Some(huaweicloud_clone); - } - } - TierType::Azure => { - if let Some(azure_) = self.azure.as_ref() { - let mut azure_clone = azure_.clone(); - azure_clone.secret_key = "REDACTED".to_string(); - azure = Some(azure_clone); - } - } - TierType::GCS => { - if let Some(gcs_) = self.gcs.as_ref() { - let mut gcs_clone = gcs_.clone(); - gcs_clone.creds = "REDACTED".to_string(); - gcs = Some(gcs_clone); - } - } - TierType::R2 => { - if let Some(r2_) = self.r2.as_ref() { - let mut r2_clone = r2_.clone(); - r2_clone.secret_key = "REDACTED".to_string(); - r2 = Some(r2_clone); - } - } - _ => (), - } - TierConfig { - version: self.version.clone(), - tier_type: self.tier_type.clone(), - name: self.name.clone(), - s3, - wasabi, - rustfs: r, - minio: compatible_backend, - aliyun, - tencent, - huaweicloud, - azure, - gcs, - r2, - } + fn clone(&self) -> Self { + self.redacted() } } impl TierConfig { + pub(crate) fn redacted(&self) -> Self { + let mut redacted = Self { + version: self.version.clone(), + tier_type: self.tier_type.clone(), + name: self.name.clone(), + ..Default::default() + }; + match self.tier_type { + TierType::S3 => { + redacted.s3 = self.s3.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + if !backend.aws_role_web_identity_token_file.is_empty() { + backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string(); + } + backend + }); + } + TierType::Wasabi => { + redacted.wasabi = self.wasabi.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::RustFS => { + redacted.rustfs = self.rustfs.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::MinIO => { + redacted.minio = self.minio.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Aliyun => { + redacted.aliyun = self.aliyun.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Tencent => { + redacted.tencent = self.tencent.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Huaweicloud => { + redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Azure => { + redacted.azure = self.azure.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + if !backend.sp_auth.client_secret.is_empty() { + backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string(); + } + backend + }); + } + TierType::GCS => { + redacted.gcs = self.gcs.clone().map(|mut backend| { + backend.creds = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::R2 => { + redacted.r2 = self.r2.clone().map(|mut backend| { + backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string(); + backend + }); + } + TierType::Unsupported => {} + } + redacted + } + pub(crate) fn clone_with_credentials(&self) -> Self { Self { version: self.version.clone(), @@ -372,31 +356,61 @@ impl TierConfig { } } +impl std::fmt::Debug for TierConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let redacted = self.redacted(); + f.debug_struct("TierConfig") + .field("version", &redacted.version) + .field("tier_type", &redacted.tier_type) + .field("name", &redacted.name) + .field("s3", &redacted.s3) + .field("wasabi", &redacted.wasabi) + .field("aliyun", &redacted.aliyun) + .field("tencent", &redacted.tencent) + .field("huaweicloud", &redacted.huaweicloud) + .field("azure", &redacted.azure) + .field("gcs", &redacted.gcs) + .field("r2", &redacted.r2) + .field("rustfs", &redacted.rustfs) + .field("minio", &redacted.minio) + .finish() + } +} + //type S3Options = impl Fn(TierS3) -> Pin>> + Send + Sync + 'static; #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierS3 { + #[serde(alias = "Name")] pub name: String, + #[serde(alias = "Endpoint")] pub endpoint: String, - #[serde(rename = "accessKey")] + #[serde(rename = "accessKey", alias = "AccessKey")] pub access_key: String, - #[serde(rename = "secretKey")] + #[serde(rename = "secretKey", alias = "SecretKey")] pub secret_key: String, + #[serde(alias = "Bucket")] pub bucket: String, + #[serde(alias = "Prefix")] pub prefix: String, + #[serde(alias = "Region")] pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, - #[serde(skip)] + #[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)] pub aws_role: bool, - #[serde(skip)] + #[serde( + rename = "AWSRoleWebIdentityTokenFile", + alias = "awsRoleWebIdentityTokenFile", + skip_serializing + )] pub aws_role_web_identity_token_file: String, - #[serde(skip)] + #[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)] pub aws_role_arn: String, - #[serde(skip)] + #[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)] pub aws_role_session_name: String, - #[serde(skip)] + #[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)] pub aws_role_duration_seconds: i32, } @@ -623,8 +637,11 @@ pub struct TierHuaweicloud { #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct ServicePrincipalAuth { + #[serde(alias = "TenantID")] pub tenant_id: String, + #[serde(alias = "ClientID")] pub client_id: String, + #[serde(alias = "ClientSecret")] pub client_secret: String, } @@ -640,9 +657,9 @@ pub struct TierAzure { pub bucket: String, pub prefix: String, pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, - #[serde(rename = "spAuth")] + #[serde(rename = "spAuth", alias = "SPAuth")] pub sp_auth: ServicePrincipalAuth, } @@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error { #[derive(Serialize, Deserialize, Default, Debug, Clone)] #[serde(default)] pub struct TierGCS { + #[serde(alias = "Name")] pub name: String, + #[serde(alias = "Endpoint")] pub endpoint: String, - #[serde(rename = "creds")] + #[serde(rename = "creds", alias = "Creds")] pub creds: String, + #[serde(alias = "Bucket")] pub bucket: String, + #[serde(alias = "Prefix")] pub prefix: String, + #[serde(alias = "Region")] pub region: String, - #[serde(rename = "storageClass")] + #[serde(rename = "storageClass", alias = "StorageClass")] pub storage_class: String, } @@ -725,6 +747,43 @@ pub struct TierR2 { mod tests { use super::*; + #[test] + fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() { + let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode"); + let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode"); + assert!(matches!(s3, TierType::S3)); + assert!(matches!(gcs, TierType::GCS)); + assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#); + assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#); + } + + #[test] + fn azure_service_principal_accepts_canonical_madmin_field_names() { + for field in ["TenantID", "ClientID", "ClientSecret"] { + let mut sp_auth = serde_json::Map::new(); + sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string())); + let config: TierConfig = serde_json::from_value(serde_json::json!({ + "type": "azure", + "Name": "COLD-AZURE", + "azure": { + "name": "COLD-AZURE", + "endpoint": "https://azure.example.invalid", + "accessKey": "account", + "secretKey": "key", + "bucket": "archive", + "SPAuth": sp_auth + } + })) + .expect("mixed RustFS/madmin Azure payload should decode"); + let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth; + + assert!( + !sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(), + "canonical {field} must not be silently discarded" + ); + } + } + fn wasabi_config() -> TierWasabi { TierWasabi { name: "COLD-WASABI".to_string(), @@ -838,9 +897,14 @@ mod tests { let config = TierConfig { tier_type: TierType::Wasabi, wasabi: Some(wasabi_config()), + rustfs: Some(TierRustFS { + access_key: "inactive-access".to_string(), + secret_key: "inactive-secret".to_string(), + ..Default::default() + }), ..Default::default() }; - let redacted = config.clone(); + let redacted = config.redacted(); assert_eq!( redacted .wasabi @@ -849,21 +913,46 @@ mod tests { .secret_key, "REDACTED" ); + assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider"); + let cloned = config.clone(); + assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED"); + assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics"); + let preserved = config.clone_with_credentials(); assert_eq!( - config - .clone_with_credentials() + preserved .wasabi .as_ref() - .expect("credential-bearing Wasabi payload should remain") + .expect("credential-bearing Wasabi snapshot should remain") .secret_key, "secret" ); + assert_eq!( + preserved + .rustfs + .expect("credential-bearing snapshots should preserve inactive provider data") + .secret_key, + "inactive-secret" + ); let mut debug_config = wasabi_config(); debug_config.secret_key = "wasabi-debug-secret-value".to_string(); let debug = format!("{debug_config:?}"); assert!(debug.contains("REDACTED")); assert!(!debug.contains("wasabi-debug-secret-value")); + + let debug = format!( + "{:?}", + TierConfig { + tier_type: TierType::RustFS, + rustfs: Some(TierRustFS { + secret_key: "rustfs-debug-secret-value".to_string(), + ..Default::default() + }), + ..Default::default() + } + ); + assert!(debug.contains("REDACTED")); + assert!(!debug.contains("rustfs-debug-secret-value")); } #[test] @@ -894,7 +983,7 @@ mod tests { assert_eq!(encoded, expected); let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode"); assert!(matches!(decoded.tier_type, TierType::Wasabi)); - let redacted = config.clone(); + let redacted = config.redacted(); assert_eq!( config .wasabi @@ -916,4 +1005,87 @@ mod tests { "REDACTED" ); } + + #[test] + fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() { + let cases = [ + ( + "s3", + TierConfig { + tier_type: TierType::S3, + s3: Some(TierS3 { + secret_key: "s3-secret-bytes".to_string(), + aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(), + ..Default::default() + }), + ..Default::default() + }, + vec!["s3-secret-bytes", "/var/run/s3-private-token"], + ), + ( + "gcs", + TierConfig { + tier_type: TierType::GCS, + gcs: Some(TierGCS { + creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(), + ..Default::default() + }), + ..Default::default() + }, + vec!["gcs-private-key-bytes"], + ), + ( + "azure", + TierConfig { + tier_type: TierType::Azure, + azure: Some(TierAzure { + secret_key: "azure-account-secret-bytes".to_string(), + sp_auth: ServicePrincipalAuth { + client_secret: "azure-client-secret-bytes".to_string(), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }, + vec!["azure-account-secret-bytes", "azure-client-secret-bytes"], + ), + ]; + + for (provider, config, secrets) in cases { + let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize"); + let debug = format!("{config:?}"); + assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted"); + assert!( + debug.contains(TIER_CREDENTIAL_REDACTED), + "{provider} Debug output should be visibly redacted" + ); + for secret in secrets { + assert!(!api.contains(secret), "{provider} API output exposed credential bytes"); + assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes"); + } + } + } + + #[test] + fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() { + let config = TierConfig { + tier_type: TierType::Azure, + azure: Some(TierAzure { + secret_key: "azure-account-secret-bytes".to_string(), + sp_auth: ServicePrincipalAuth::default(), + ..Default::default() + }), + ..Default::default() + }; + + let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize"); + let debug = format!("{config:?}"); + + assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED); + assert_eq!(api["azure"]["spAuth"]["client_secret"], ""); + assert!(debug.contains("client_secret: \"\"")); + assert!(!debug.contains("client_secret: \"REDACTED\"")); + assert!(!debug.contains("azure-account-secret-bytes")); + } } diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 8eb6e4058..ee48c116c 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -20,7 +20,7 @@ use crate::error::is_err_bucket_not_found; use crate::services::tier::{ - tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, + tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR}, warm_backend_aliyun::WarmBackendAliyun, @@ -55,18 +55,21 @@ use s3s::header::{ }; use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use time::OffsetDateTime; use time::format_description::well_known::{Rfc2822, Rfc3339}; +use tokio::io::AsyncReadExt; use tracing::{info, warn}; pub type WarmBackendImpl = Box; -const PROBE_OBJECT: &str = "probeobject"; - /// Largest object the S3-compatible warm backends accept for a multipart put. pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; /// Part-count ceiling S3-compatible services impose on a multipart upload. pub(crate) const MAX_PARTS_COUNT: i64 = 10000; +pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30); +const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1); +const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1); #[derive(Default)] pub struct WarmBackendGetOpts { @@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> { pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>, } +/// Return the authority format accepted by `TransitionClient::new` while +/// retaining an explicitly configured port. `url::Url::host_str()` omits the +/// brackets needed when an IPv6 literal is combined with a port. +pub(crate) fn endpoint_authority(url: &url::Url) -> Result { + let host = url + .host_str() + .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; + let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 }); + if host.starts_with('[') && host.ends_with(']') { + Ok(format!("{host}:{port}")) + } else if host.contains(':') { + Ok(format!("[{host}]:{port}")) + } else { + Ok(format!("{host}:{port}")) + } +} + /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers. /// /// Credential, bucket, and endpoint validation run in this order because the @@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend( bucket_lookup: params.bucket_lookup, ..Default::default() }; - let scheme = u.scheme(); - let default_port = if scheme == "https" { 443 } else { 80 }; - let host = u - .host_str() - .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; - // Runs after the host-presence check above (not immediately after Url::parse) so a - // host-less endpoint still reports this constructor's own "missing host" text instead of - // validate_endpoint's differently-worded rejection for the same input. + let endpoint = endpoint_authority(&u)?; + // Run the SSRF guard after the host-presence check so a host-less endpoint + // keeps this constructor's stable error text. (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; - let client = - TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?; + let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); @@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler { } } -pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { - let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; - w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?; - let remote_version_id = w - .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5) - .await - .map_err(|_| ERR_TIER_PERM_ERR.clone())?; +async fn remove_discovered_probe_candidate( + w: &WarmBackendImpl, + probe_object: &str, + candidate: TransitionCandidateProbe, +) -> Result { + match candidate { + TransitionCandidateProbe::Missing => Ok(false), + TransitionCandidateProbe::VersionedPresent(remote_version_id) => { + w.remove_exact(probe_object, &remote_version_id).await?; + Ok(true) + } + TransitionCandidateProbe::UnversionedPresent => { + w.remove(probe_object, "").await?; + Ok(true) + } + TransitionCandidateProbe::Ambiguous => { + Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions")) + } + TransitionCandidateProbe::Unsupported => Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot discover the outcome of a probe PUT", + )), + } +} - if w.validate_remote_version_id(&remote_version_id).is_err() { - w.remove_exact(PROBE_OBJECT, &remote_version_id) +async fn compensate_uncertain_probe_put( + w: &WarmBackendImpl, + probe_object: &str, + settle_deadline: tokio::time::Instant, +) -> Result<(), std::io::Error> { + let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT; + let mut removed_any = false; + while tokio::time::Instant::now() < settle_deadline { + let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await { + Ok(candidate) => candidate?, + Err(_) => break, + }; + if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any { + break; + } + removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate)) .await - .map_err(|_| ERR_TIER_PERM_ERR.clone())?; - return Err(ERR_TIER_INVALID_CONFIG.clone()); + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??; + + let now = tokio::time::Instant::now(); + if now >= settle_deadline { + break; + } + tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await; } - let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await; - let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await; - //xhttp.DrainBody(r); - if read_result.is_err() || remove_result.is_err() { + let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object)) + .await + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state") + })??; + if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate)) + .await + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate") + })?? + { + return Ok(()); + } + + let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??; + match final_candidate { + TransitionCandidateProbe::Missing => Ok(()), + _ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")), + } +} + +fn probe_cleanup_incomplete_error() -> AdminError { + let mut err = ERR_TIER_PERM_ERR.clone(); + err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string(); + err +} + +async fn check_warm_backend_with_deadlines( + w: Option<&WarmBackendImpl>, + deadline: tokio::time::Instant, + cleanup_deadline: tokio::time::Instant, +) -> Result<(), AdminError> { + let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; + let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4()); + let timeout_error = || { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out validating the remote tier mutation".to_string(); + err + }; + tokio::time::timeout_at(deadline, w.validate()) + .await + .map_err(|_| timeout_error())? + .map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?; + let put_result = + tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await; + let remote_version_id = match put_result { + Ok(Ok(remote_version_id)) => remote_version_id, + Ok(Err(_)) => { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_PERM_ERR.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + Err(_) => { + let err = timeout_error(); + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => err, + Err(_) => probe_cleanup_incomplete_error(), + }); + } + }; + + // S3-family backends do not replay a failed request before returning `Ok`, + // while GCS discovers every matching generation. The authoritative probe + // below therefore closes the acknowledged-PUT path; only an error or + // timeout needs the longer visibility reconciliation above. + let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await { + Ok(Ok(candidate)) => candidate, + Ok(Err(_)) | Err(_) => { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_INVALID_CONFIG.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + }; + let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok(); + let response_matches_candidate = match &authoritative_candidate { + TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(), + TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id, + TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false, + }; + if !response_version_is_valid || !response_matches_candidate { + return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await { + Ok(()) => ERR_TIER_INVALID_CONFIG.clone(), + Err(_) => probe_cleanup_incomplete_error(), + }); + } + + let read_result = tokio::time::timeout_at(deadline, async { + let mut reader = w + .get( + &probe_object, + &remote_version_id, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + .map_err(|_| ERR_TIER_PERM_ERR.clone())?; + let mut body = Vec::new(); + reader + .take(7) + .read_to_end(&mut body) + .await + .map_err(|_| ERR_TIER_PERM_ERR.clone())?; + if body != b"RustFS" { + return Err(ERR_TIER_PERM_ERR.clone()); + } + Ok(()) + }) + .await + .map_err(|_| timeout_error()) + .and_then(|result| result); + let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async { + if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? { + return Err(std::io::Error::other("remote tier probe disappeared before cleanup")); + } + match w.probe_transition_candidate(&probe_object).await? { + TransitionCandidateProbe::Missing => Ok(()), + _ => Err(std::io::Error::other("remote tier probe remained after cleanup")), + } + }) + .await; + if !matches!(cleanup_result, Ok(Ok(()))) { + return Err(probe_cleanup_incomplete_error()); + } + if let Err(err) = read_result { //if is_err_bucket_not_found(&err) { // return Err(ERR_TIER_BUCKET_NOT_FOUND); //} @@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin return Err(ERR_TIER_MISSING_CREDENTIALS); }*/ //else { - return Err(ERR_TIER_PERM_ERR.clone()); + return Err(err); //} } Ok(()) } +/// Validate a backend using a caller-owned deadline while retaining a bounded +/// reconciliation window for an uncertain probe PUT. The validation future is +/// kept alive through cleanup so an outer timeout cannot abandon the remote +/// probe object. +pub(crate) async fn check_warm_backend_until( + w: Option<&WarmBackendImpl>, + deadline: tokio::time::Instant, +) -> Result<(), AdminError> { + check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await +} + +pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { + let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT; + check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await +} + pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { let mut d: Option = None; match tier.tier_type { @@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result, + present: Arc, removed_versions: Arc>>, + returned_version: String, } struct RecordingProbeBackend { get_versions: Arc>>, + present: Arc, removed_versions: Arc>>, + remove_clears_candidate: bool, fail_get: bool, + body: ProbeBody, + } + + struct HangingProbePutBackend { + put_started: Arc, + present: Arc, + probes: Arc, + removed_versions: Arc>>, + } + + struct LateVisibleProbeBackend { + visible_at: tokio::time::Instant, + removed: Arc, + probes: Arc, + removed_versions: Arc>>, + } + + #[derive(Clone, Copy)] + enum ProbeBody { + Exact, + Mismatch, } #[async_trait::async_trait] @@ -976,7 +1194,7 @@ mod tests { } async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { - Ok(uuid::Uuid::nil().to_string()) + Ok(self.returned_version.clone()) } async fn put_with_meta( @@ -999,10 +1217,19 @@ mod tests { } async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.present.store(false, Ordering::SeqCst); self.removed_versions.lock().await.push(rv.to_string()); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> Result { + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> Result { Ok(false) } @@ -1029,15 +1256,121 @@ mod tests { if self.fail_get { Err(std::io::Error::other("probe GET failed")) } else { - Ok(ReadCloser::new(std::io::Cursor::new(Vec::new()))) + match self.body { + ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))), + ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))), + } } } async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + if self.remove_clears_candidate { + self.present.store(false, Ordering::SeqCst); + } self.removed_versions.lock().await.push(rv.to_string()); Ok(()) } + async fn probe_transition_candidate(&self, _object: &str) -> Result { + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> Result { + Ok(false) + } + } + + #[async_trait::async_trait] + impl WarmBackend for HangingProbePutBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { + self.put_started.notify_one(); + std::future::pending().await + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result { + Err(std::io::Error::other("GET must not run after a timed out probe PUT")) + } + + async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> { + Err(std::io::Error::other("generic remove must not replace exact probe cleanup")) + } + + async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.present.store(false, Ordering::SeqCst); + self.removed_versions.lock().await.push(rv.to_string()); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> Result { + self.probes.fetch_add(1, Ordering::SeqCst); + if self.present.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + + async fn in_use(&self) -> Result { + Ok(false) + } + } + + #[async_trait::async_trait] + impl WarmBackend for LateVisibleProbeBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result { + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "probe PUT response was lost before the object became visible", + )) + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result { + Err(std::io::Error::other("GET must not run after a lost probe PUT response")) + } + + async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> { + Err(std::io::Error::other("generic remove must not replace exact probe cleanup")) + } + + async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> { + self.removed.store(true, Ordering::SeqCst); + self.removed_versions.lock().await.push(rv.to_string()); + Ok(()) + } + + async fn probe_transition_candidate(&self, _object: &str) -> Result { + self.probes.fetch_add(1, Ordering::SeqCst); + if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) { + Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string())) + } else { + Ok(TransitionCandidateProbe::Missing) + } + } + async fn in_use(&self) -> Result { Ok(false) } @@ -1098,13 +1431,15 @@ mod tests { assert_eq!(probe, TransitionCandidateProbe::Unsupported); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() { let gets = Arc::new(AtomicUsize::new(0)); let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend { gets: gets.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + returned_version: uuid::Uuid::nil().to_string(), }); let err = check_warm_backend(Some(&backend)) @@ -1113,7 +1448,27 @@ mod tests { assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); assert_eq!(gets.load(Ordering::SeqCst), 0); - assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() { + let gets = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend { + gets: gets.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + returned_version: String::new(), + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("an empty PUT version must not read or generically delete a versioned object"); + + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert_eq!(gets.load(Ordering::SeqCst), 0); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); } #[tokio::test] @@ -1122,8 +1477,11 @@ mod tests { let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { get_versions: get_versions.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + remove_clears_candidate: true, fail_get: false, + body: ProbeBody::Exact, }); check_warm_backend(Some(&backend)) @@ -1140,8 +1498,11 @@ mod tests { let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { get_versions: get_versions.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), removed_versions: removed_versions.clone(), + remove_clears_candidate: true, fail_get: true, + body: ProbeBody::Exact, }); let err = check_warm_backend(Some(&backend)) @@ -1153,6 +1514,169 @@ mod tests { assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); } + #[tokio::test] + async fn check_warm_backend_removes_probe_after_body_mismatch() { + let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { + get_versions, + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + remove_clears_candidate: true, + fail_get: false, + body: ProbeBody::Mismatch, + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("a mismatched body should fail after cleanup"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test] + async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() { + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let backend: WarmBackendImpl = Box::new(RecordingProbeBackend { + get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + removed_versions: removed_versions.clone(), + remove_clears_candidate: false, + fail_get: false, + body: ProbeBody::Exact, + }); + + let err = check_warm_backend(Some(&backend)) + .await + .expect_err("cleanup must not succeed while the deleted candidate remains visible"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_reconciles_a_lost_put_response() { + let backend = MockWarmBackend::new(); + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("a lost probe PUT response must fail after compensation"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.exact_remove_count(), 1); + let operations = backend.op_log().await; + let put = operations.iter().find_map(|operation| match operation { + MockWarmOp::Put { object } => Some(object), + _ => None, + }); + let probe = operations.iter().find_map(|operation| match operation { + MockWarmOp::Probe { object } => Some(object), + _ => None, + }); + let remove = operations.iter().find_map(|operation| match operation { + MockWarmOp::Remove { object } => Some(object), + _ => None, + }); + let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else { + panic!("lost-response compensation should PUT, probe, and remove"); + }; + assert_eq!(put, probe); + assert_eq!(probe, remove); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_retries_until_a_late_put_becomes_visible() { + let probes = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend { + visible_at: tokio::time::Instant::now() + Duration::from_secs(5), + removed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + probes: probes.clone(), + removed_versions: removed_versions.clone(), + }); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("a late-visible probe PUT must still report the lost response"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!( + probes.load(Ordering::SeqCst) > 5, + "reconciliation must not stop at the first Missing result" + ); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + + #[tokio::test] + async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() { + for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] { + let backend = MockWarmBackend::new(); + backend.set_transition_candidate_probe_override(Some(candidate)).await; + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("an uncertain candidate must fail without a guessed delete"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(backend.remove_count().await, 0); + assert_eq!(backend.object_count().await, 1); + } + } + + #[tokio::test] + async fn check_warm_backend_reports_an_exact_cleanup_failure() { + let backend = MockWarmBackend::new(); + backend.set_remove_failure(true); + backend.lose_next_put_response(); + let driver: WarmBackendImpl = Box::new(backend.clone()); + + let err = check_warm_backend(Some(&driver)) + .await + .expect_err("an exact cleanup failure must replace the ambiguous PUT error"); + + assert_eq!(err.code, ERR_TIER_PERM_ERR.code); + assert!(err.message.contains("cleanup is incomplete")); + assert_eq!(backend.exact_remove_count(), 1); + assert_eq!(backend.object_count().await, 1); + } + + #[tokio::test(start_paused = true)] + async fn check_warm_backend_reconciles_a_timed_out_put() { + let put_started = Arc::new(tokio::sync::Notify::new()); + let probes = Arc::new(AtomicUsize::new(0)); + let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let driver: WarmBackendImpl = Box::new(HangingProbePutBackend { + put_started: put_started.clone(), + present: Arc::new(std::sync::atomic::AtomicBool::new(true)), + probes: probes.clone(), + removed_versions: removed_versions.clone(), + }); + let check = check_warm_backend(Some(&driver)); + tokio::pin!(check); + tokio::select! { + _ = put_started.notified() => {} + result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"), + } + + tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await; + let err = check.await.expect_err("a timed out probe PUT must fail after compensation"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!( + probes.load(Ordering::SeqCst) > 1, + "timed-out PUT reconciliation must keep checking through the visibility window" + ); + assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]); + } + #[tokio::test] async fn new_wasabi_backend_honors_probe_flag() { let tier = TierConfig { @@ -1298,6 +1822,15 @@ mod tests { assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80)); } + #[test] + fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() { + let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse"); + assert_eq!( + endpoint_authority(&url).expect("the endpoint should have an authority"), + "[2001:db8::1]:9443" + ); + } + #[tokio::test] async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() { let mut params = s3_compatible_params("http://tier.example.com:9000"); diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index e848f4f29..27fb7decd 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierAliyun, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index 7b3723eac..ed9ef0cf8 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierAzure, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + // Azure currently uses the shared S3/SigV4 transport, but its normal + // object path cannot persist exact remote versions across mixed + // RustFS releases. The mutation probe may still detect and precisely + // remove a versioned test object before rejecting that configuration. + self.0 + .probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id") + .await + } + async fn in_use(&self) -> Result { self.0.in_use().await } @@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure { mod tests { use super::*; use crate::services::tier::tier_config::TierAzure; + use rustfs_s3_client::{ + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, TransitionClient, TransitionCore}, + }; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String { + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut buffer).await.expect("fixture request should be readable"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + } + String::from_utf8_lossy(&request).into_owned() + } /// The SSRF guard itself is exercised once, generically, in /// `warm_backend::tests` (see backlog#2040/backlog#2041 and @@ -119,4 +146,85 @@ mod tests { Err(err) => assert!(err.to_string().contains("not allowed")), } } + + #[tokio::test] + async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET"); + let get_request = read_request_head(&mut get_stream).await; + get_stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx", + ) + .await + .expect("fixture should write candidate GET response"); + + let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE"); + let delete_request = read_request_head(&mut delete_stream).await; + delete_stream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("fixture should write exact DELETE response"); + (get_request, delete_request) + }); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "azure", + ) + .await + .expect("fixture client should build"), + ); + let backend = WarmBackendAzure(WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }); + assert!( + !backend.0.client.provider_version_capabilities().exact_get_delete, + "probe-only version discovery must not change Azure's persisted data-path contract" + ); + + let candidate = backend + .probe_transition_candidate("probe") + .await + .expect("Azure candidate should be discovered"); + assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string())); + backend + .remove_exact("probe", "azure-version") + .await + .expect("Azure candidate should be deleted by exact version"); + + let (get_request, delete_request) = fixture.await.expect("fixture should join"); + assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n")); + assert!( + delete_request + .lines() + .next() + .is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version ")) + ); + } } diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 63c9a62c1..03c2df988 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -18,13 +18,13 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::io::{Error, ErrorKind}; use std::sync::Arc; use bytes::Bytes; -use google_cloud_auth::credentials::Credentials; -use google_cloud_auth::credentials::user_account::Builder; +use google_cloud_auth::credentials::service_account::Builder; use google_cloud_storage as gcs; use google_cloud_storage::client::Storage; use google_cloud_storage::client::StorageControl; @@ -32,7 +32,7 @@ use std::convert::TryFrom; use crate::services::tier::{ tier_config::TierGCS, - warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}, }; use rustfs_s3_client::{ admin_handler_utils::AdminError, @@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MAX_GCS_CANDIDATE_PAGES: usize = 64; fn parse_generation(remote_version: &str) -> Result, Error> { if remote_version.is_empty() { @@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result, Error> { Ok(Some(generation)) } +fn append_gcs_chunk( + contents: &mut Vec, + chunk: Result, + max_response_bytes: Option, +) -> std::io::Result<()> { + let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?; + if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS object response exceeded the configured byte limit", + )); + } + contents.extend_from_slice(&chunk); + Ok(()) +} + +fn gcs_bucket_resource_name(bucket: &str) -> String { + format!("projects/_/buckets/{bucket}") +} + +struct GcsCandidateObject { + name: String, + generation: i64, +} + +struct GcsCandidatePage { + objects: Vec, + next_page_token: String, +} + +async fn probe_exact_gcs_candidate( + remote_object: &str, + mut fetch_page: F, +) -> Result +where + F: FnMut(String) -> Fut, + Fut: Future>, +{ + let mut page_token = String::new(); + let mut seen_page_tokens = HashSet::new(); + let mut generation = None; + let mut pages_seen = 0_usize; + + loop { + pages_seen += 1; + if pages_seen > MAX_GCS_CANDIDATE_PAGES { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS candidate listing exceeded the page limit", + )); + } + let response = fetch_page(page_token.clone()).await?; + for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) { + if candidate.generation <= 0 { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "GCS candidate listing returned a non-positive generation", + )); + } + if generation.replace(candidate.generation).is_some() { + return Ok(TransitionCandidateProbe::Ambiguous); + } + } + + if response.next_page_token.is_empty() { + break; + } + if !seen_page_tokens.insert(response.next_page_token.clone()) { + return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token")); + } + page_token = response.next_page_token; + } + + Ok(match generation { + Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()), + None => TransitionCandidateProbe::Missing, + }) +} + pub struct WarmBackendGCS { pub client: Arc, pub control: Arc, @@ -80,8 +160,8 @@ impl WarmBackendGCS { .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; } - let authorized_user = serde_json::from_str(&conf.creds)?; - let credentials = Builder::new(authorized_user) + let service_account = serde_json::from_str(&conf.creds)?; + let credentials = Builder::new(service_account) //.with_retry_policy(AlwaysRetry.with_attempt_limit(3)) //.with_backoff_policy(backoff) .build() @@ -98,7 +178,11 @@ impl WarmBackendGCS { let client = Arc::new(client); // Control-plane client: the data-plane `Storage` client cannot delete or list objects; // delete_object/list_objects live on StorageControl. - let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else { + let mut control_builder = StorageControl::builder().with_credentials(credentials); + if !conf.endpoint.is_empty() { + control_builder = control_builder.with_endpoint(conf.endpoint.clone()); + } + let Ok(control) = control_builder.build().await else { return Err(std::io::Error::other("StorageControl::builder error")); }; let control = Arc::new(control); @@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS { ReaderImpl::Body(content_body) => content_body.to_vec(), ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?, }; + let bucket = gcs_bucket_resource_name(&self.bucket); let Ok(res) = Box::pin( self.client - .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d)) + .write_object(&bucket, &self.get_dest(object), Bytes::from(d)) .send_buffered(), ) .await @@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS { } async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { - let mut req = self.client.read_object(&self.bucket, &self.get_dest(object)); + let bucket = gcs_bucket_resource_name(&self.bucket); + let mut req = self.client.read_object(&bucket, &self.get_dest(object)); + let mut max_response_bytes = None; if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); } @@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS { .length .try_into() .map_err(|_| std::io::Error::other("invalid range: negative length"))?; + max_response_bytes = Some( + opts.length + .try_into() + .map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?, + ); req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count)); } @@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS { return Err(std::io::Error::other("read_object error")); }; let mut contents = Vec::new(); - while let Ok(Some(chunk)) = reader.next().await.transpose() { - contents.extend_from_slice(&chunk); + while let Some(chunk) = reader.next().await { + append_gcs_chunk(&mut contents, chunk, max_response_bytes)?; } Ok(ReadCloser::new(std::io::Cursor::new(contents))) } @@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS { let mut req = self .control .delete_object() - .set_bucket(format!("projects/_/buckets/{}", self.bucket)) + .set_bucket(gcs_bucket_resource_name(&self.bucket)) .set_object(self.get_dest(object)); if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); @@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS { Ok(()) } + async fn probe_transition_candidate(&self, object: &str) -> Result { + let remote_object = self.get_dest(object); + let parent = gcs_bucket_resource_name(&self.bucket); + probe_exact_gcs_candidate(&remote_object, |page_token| { + let control = self.control.clone(); + let parent = parent.clone(); + let prefix = remote_object.clone(); + async move { + let response = control + .list_objects() + .set_parent(parent) + .set_prefix(prefix) + .set_versions(true) + .set_page_size(2) + .set_page_token(page_token) + .send() + .await + .map_err(|err| std::io::Error::other(err.to_string()))?; + Ok(GcsCandidatePage { + objects: response + .objects + .into_iter() + .map(|candidate| GcsCandidateObject { + name: candidate.name, + generation: candidate.generation, + }) + .collect(), + next_page_token: response.next_page_token, + }) + } + }) + .await + } + async fn in_use(&self) -> Result { // Scope the listing to this tier's prefix (matching the other warm backends) and only // need to know whether a single object exists. let resp = self .control .list_objects() - .set_parent(format!("projects/_/buckets/{}", self.bucket)) + .set_parent(gcs_bucket_resource_name(&self.bucket)) .set_prefix(self.prefix.clone()) .set_page_size(1) .send() @@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS { #[cfg(test)] mod tests { + use super::GcsCandidateObject; + use super::GcsCandidatePage; + use super::MAX_GCS_CANDIDATE_PAGES; use super::WarmBackendGCS; + use super::append_gcs_chunk; + use super::gcs_bucket_resource_name; use super::parse_generation; + use super::probe_exact_gcs_candidate; use crate::services::tier::tier_config::TierGCS; + use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}; + use google_cloud_auth::credentials::anonymous::Builder as Anonymous; + use google_cloud_storage::client::{Storage, StorageControl}; use std::io::ErrorKind; + use std::sync::Arc; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + async fn serve_data_plane_fixture(listener: TcpListener) -> Vec { + let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#; + let responses = [ + format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}", + upload_body.len() + ), + "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!" + .to_string(), + "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!" + .to_string(), + ]; + let mut requests = Vec::new(); + + for response in responses { + let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request"); + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 1024]; + let count = stream + .read(&mut chunk) + .await + .expect("the GCS fixture should read request headers"); + if count == 0 { + break; + } + request.extend_from_slice(&chunk[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let header_end = request + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("the GCS fixture should receive complete request headers"); + let headers = String::from_utf8_lossy(&request[..header_end]); + if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) { + stream + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .await + .expect("the GCS fixture should acknowledge 100-continue"); + } + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("content-length should be numeric")) + }) + .unwrap_or_default(); + while request.len() < header_end.saturating_add(content_length) { + let mut chunk = [0_u8; 1024]; + let count = stream + .read(&mut chunk) + .await + .expect("the GCS fixture should read the request body"); + if count == 0 { + break; + } + request.extend_from_slice(&chunk[..count]); + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + stream + .write_all(response.as_bytes()) + .await + .expect("the GCS fixture should write its response"); + } + + requests + } + + fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage { + GcsCandidatePage { + objects: objects + .iter() + .map(|(name, generation)| GcsCandidateObject { + name: (*name).to_string(), + generation: *generation, + }) + .collect(), + next_page_token: next_page_token.to_string(), + } + } + + async fn probe_candidate_pages( + remote_object: &str, + pages: Vec, + ) -> (Result, Vec) { + let mut pages = pages.into_iter(); + let mut requested_tokens = Vec::new(); + let result = probe_exact_gcs_candidate(remote_object, |page_token| { + requested_tokens.push(page_token); + std::future::ready( + pages + .next() + .ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")), + ) + }) + .await; + (result, requested_tokens) + } #[test] fn generation_parser_preserves_exact_numeric_versions() { @@ -241,6 +483,243 @@ mod tests { } } + #[test] + fn body_collection_propagates_an_error_after_a_complete_prefix() { + let mut contents = Vec::new(); + append_gcs_chunk::(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7)) + .expect("the prefix chunk should be collected"); + let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7)) + .expect_err("a trailing stream error must not be mistaken for EOF"); + + assert_eq!(contents, b"RustFS"); + assert!(err.to_string().contains("trailing stream failure")); + } + + #[test] + fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() { + let mut contents = Vec::new(); + let err = append_gcs_chunk::(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7)) + .expect_err("the GCS collection layer must reject an oversized probe response"); + + assert!(contents.is_empty()); + assert_eq!(err.kind(), ErrorKind::InvalidData); + } + + #[tokio::test] + async fn candidate_probe_finds_exact_name_on_first_or_later_page() { + let (first, first_tokens) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await; + assert_eq!( + first.expect("an exact first-page object should be discovered"), + TransitionCandidateProbe::VersionedPresent("7".to_string()) + ); + assert_eq!(first_tokens, [""]); + + let (later, later_tokens) = probe_candidate_pages( + "prefix/object", + vec![ + candidate_page(&[("prefix/object-shadow", 8)], "next"), + candidate_page(&[("prefix/object", 9)], ""), + ], + ) + .await; + assert_eq!( + later.expect("an exact later-page object should be discovered"), + TransitionCandidateProbe::VersionedPresent("9".to_string()) + ); + assert_eq!(later_tokens, ["", "next"]); + } + + #[tokio::test] + async fn candidate_probe_ignores_non_exact_prefix_matches() { + let (probe, _) = probe_candidate_pages( + "prefix/object", + vec![candidate_page( + &[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)], + "", + )], + ) + .await; + + assert_eq!( + probe.expect("prefix-only matches should not hide the exact object"), + TransitionCandidateProbe::VersionedPresent("7".to_string()) + ); + } + + #[tokio::test] + async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() { + let (probe, _) = probe_candidate_pages( + "prefix/object", + vec![ + candidate_page(&[("prefix/object", 7)], "next"), + candidate_page(&[("prefix/object", 8)], ""), + ], + ) + .await; + + assert_eq!( + probe.expect("multiple exact generations should produce a conservative result"), + TransitionCandidateProbe::Ambiguous + ); + } + + #[tokio::test] + async fn candidate_probe_reports_missing_without_an_exact_name() { + let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await; + + assert_eq!( + probe.expect("a complete listing without an exact name should be definitive"), + TransitionCandidateProbe::Missing + ); + } + + #[tokio::test] + async fn candidate_probe_rejects_non_positive_generations() { + for generation in [0, -1] { + let (probe, _) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await; + let err = probe.expect_err("a non-positive GCS generation must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}"); + } + } + + #[tokio::test] + async fn candidate_probe_rejects_a_page_token_that_does_not_advance() { + let (probe, requested_tokens) = + probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await; + + let err = probe.expect_err("a repeated GCS page token must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens, ["", "next"]); + } + + #[tokio::test] + async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() { + let (probe, requested_tokens) = probe_candidate_pages( + "prefix/object", + vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")], + ) + .await; + + let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens, ["", "a", "b"]); + } + + #[tokio::test] + async fn candidate_probe_rejects_an_unbounded_unique_token_chain() { + let pages = (0..MAX_GCS_CANDIDATE_PAGES) + .map(|index| candidate_page(&[], &format!("token-{index}"))) + .collect(); + let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await; + + let err = probe.expect_err("an unbounded unique page-token chain must fail closed"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES); + } + + #[tokio::test] + async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("the GCS fixture should bind a loopback port"); + let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address")); + let fixture = tokio::spawn(serve_data_plane_fixture(listener)); + let credentials = Anonymous::new().build(); + let client = Storage::builder() + .with_endpoint(endpoint.clone()) + .with_credentials(credentials.clone()) + .build() + .await + .expect("the GCS data client should build"); + let control = StorageControl::builder() + .with_endpoint(endpoint) + .with_credentials(credentials) + .build() + .await + .expect("the GCS control client should build"); + let backend = WarmBackendGCS { + client: Arc::new(client), + control: Arc::new(control), + bucket: "tier-bucket".to_string(), + prefix: String::new(), + }; + + let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async { + let version = backend + .put( + "probe", + rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")), + 6, + ) + .await + .expect("a plain configured bucket should reach the GCS upload transport"); + let mut reader = backend + .get( + "probe", + &version, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + .expect("a plain configured bucket should reach the GCS read transport"); + let mut body = Vec::new(); + reader + .read_to_end(&mut body) + .await + .expect("the fixture body should be readable"); + let oversized_error = match backend + .get( + "probe", + &version, + WarmBackendGetOpts { + start_offset: 0, + length: 7, + }, + ) + .await + { + Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"), + Err(err) => err, + }; + let requests = fixture.await.expect("the GCS fixture task should finish"); + (version, body, oversized_error.kind(), requests) + }) + .await + .expect("the GCS data-plane requests should not be rejected before transport"); + + assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket"); + assert_eq!(version, "123"); + assert_eq!(body, b"RustFS!"); + assert_eq!(oversized_error_kind, ErrorKind::InvalidData); + assert!( + requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"), + "unexpected upload request line: {}", + requests[0].lines().next().unwrap_or_default() + ); + assert!( + requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"), + "unexpected read request line: {}", + requests[1].lines().next().unwrap_or_default() + ); + assert!( + requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"), + "the GCS probe read must preserve its seven-byte range" + ); + assert!( + requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"), + "unexpected oversized read request line: {}", + requests[2].lines().next().unwrap_or_default() + ); + assert!( + requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"), + "the oversized response must be fetched under the same seven-byte request boundary" + ); + } + #[tokio::test] async fn new_rejects_loopback_endpoint_before_credential_setup() { let conf = TierGCS { diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index f4f489e9c..73ced2cef 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierHuaweicloud, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 55a29418c..5462fc52c 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -26,11 +26,12 @@ use crate::services::tier::{ tier_config::TierS3, warm_backend::{ TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, - build_transition_put_options, + build_transition_put_options, endpoint_authority, }, }; use http::HeaderMap; use rustfs_s3_client::{ + api_error_response::to_error_response, api_get_options::GetObjectOptions, api_list::ListObjectsOptions, api_put_object::PutObjectOptions, @@ -43,7 +44,7 @@ use rustfs_s3_client::{ }; use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::path::SLASH_SEPARATOR; -use s3s::dto::BucketVersioningStatus; +use s3s::{S3ErrorCode, dto::BucketVersioningStatus}; pub struct WarmBackendS3 { pub client: Arc, @@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result Result, std::io::Error> { + if opts.start_offset < 0 || opts.length <= 0 { + return Ok(None); + } + usize::try_from(opts.length) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?; + let end_offset = opts + .start_offset + .checked_add(opts.length - 1) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?; + Ok(Some((opts.start_offset, end_offset))) +} + impl WarmBackendS3 { pub async fn new(conf: &TierS3, _tier: &str) -> Result { Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await @@ -132,10 +146,8 @@ impl WarmBackendS3 { bucket_lookup, ..Default::default() }; - let host = u - .host() - .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?; - let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?; + let endpoint = endpoint_authority(&u)?; + let client = TransitionClient::new(&endpoint, opts, tier_type).await?; let client = Arc::new(client); let core = TransitionCore(Arc::clone(&client)); @@ -177,10 +189,8 @@ impl WarmBackendS3 { if !rv.is_empty() { gopts.version_id = rv.to_string(); } - if opts.start_offset >= 0 && opts.length > 0 { - gopts - .set_range(opts.start_offset, opts.start_offset + opts.length - 1) - .map_err(std::io::Error::other)?; + if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? { + gopts.set_range(start_offset, end_offset)?; } let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?; Ok((headers, reader)) @@ -191,36 +201,64 @@ impl WarmBackendS3 { remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str())) } - async fn probe_transition_candidate_versions( + async fn probe_current_transition_candidate_with_header( &self, object: &str, - bucket_versioning: RemoteBucketVersioning, + raw_version_header: Option<&'static str>, ) -> Result { - let remote_object = self.get_dest(object); - let mut opts = ListObjectsOptions::default(); - opts.set("prefix", &remote_object); - opts.set("max-keys", "1000"); - - let mut key_marker = String::new(); - let mut version_id_marker = String::new(); - let mut candidates = TransitionCandidateVersions::default(); - loop { - let versions = self - .client - .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "") - .await?; - candidates.extend(&remote_object, &versions); - if candidates.is_ambiguous() { - return Ok(TransitionCandidateProbe::Ambiguous); + match self + .get_with_headers( + object, + "", + WarmBackendGetOpts { + start_offset: 0, + length: 1, + }, + ) + .await + { + Ok((headers, _)) => { + let version_id = match raw_version_header { + Some(header_name) => match headers.get(header_name) { + Some(value) => { + let version_id = value.to_str().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote object version id is not valid ASCII", + ) + })?; + validate_remote_version_id(version_id)?; + Some(version_id) + } + None => None, + }, + None => self.client.raw_version_id(&headers)?, + }; + Ok(match version_id { + Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()), + None => TransitionCandidateProbe::UnversionedPresent, + }) } - if !versions.is_truncated { - return classify_transition_candidates(candidates, bucket_versioning); + Err(err) => { + let response = to_error_response(&err); + if response.code == S3ErrorCode::NoSuchKey { + Ok(TransitionCandidateProbe::Missing) + } else { + Err(err) + } } - - advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?; } } + pub(crate) async fn probe_transition_candidate_with_raw_version_header( + &self, + object: &str, + raw_version_header: &'static str, + ) -> Result { + self.probe_current_transition_candidate_with_header(object, Some(raw_version_header)) + .await + } + async fn probe_transition_candidate_identity( &self, object: &str, @@ -343,6 +381,7 @@ struct TransitionCandidateVersions { } impl TransitionCandidateVersions { + #[cfg(test)] fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) { for version in versions.versions.iter().filter(|version| version.key == remote_object) { if self.version_id.is_some() { @@ -353,10 +392,6 @@ impl TransitionCandidateVersions { } } - fn is_ambiguous(&self) -> bool { - self.ambiguous - } - fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe { if self.ambiguous { return TransitionCandidateProbe::Ambiguous; @@ -380,6 +415,8 @@ impl TransitionCandidateVersions { mod tests { use super::*; use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version}; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[tokio::test] async fn new_rejects_loopback_endpoint_before_network_setup() { @@ -398,6 +435,204 @@ mod tests { } } + #[tokio::test] + async fn new_preserves_an_explicit_endpoint_port() { + let conf = TierS3 { + endpoint: "https://tier.example.com:9443".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + let backend = WarmBackendS3::new(&conf, "tier") + .await + .expect("a well-formed S3 endpoint should initialize without network I/O"); + assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com")); + assert_eq!(backend.client.endpoint_url.port(), Some(9443)); + } + + #[tokio::test] + async fn overflowing_get_range_is_rejected_before_network_io() { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "s3", + ) + .await + .expect("fixture client should build"), + ); + let backend = WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }; + + let err = backend + .get_with_headers( + "probe", + "", + WarmBackendGetOpts { + start_offset: i64::MAX, + length: 2, + }, + ) + .await + .expect_err("an overflowing range must fail before issuing a GET"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err() + ); + } + + async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle>)> { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let fixture = tokio::spawn(async move { + let responses = [ + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", + "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", + ]; + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + stream + .write_all(response.as_bytes()) + .await + .expect("fixture should write candidate response"); + } + requests + }); + let client = Arc::new( + TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "s3", + ) + .await + .expect("fixture client should build"), + ); + Some(( + WarmBackendS3 { + core: TransitionCore(Arc::clone(&client)), + client, + bucket: "bucket".to_string(), + prefix: String::new(), + storage_class: String::new(), + }, + fixture, + )) + } + + #[tokio::test] + async fn candidate_probe_uses_only_exact_bounded_get_permissions() { + let Some((backend, fixture)) = candidate_probe_fixture().await else { + return; + }; + + assert_eq!( + backend + .probe_transition_candidate("versioned-probe") + .await + .expect("versioned candidate should be discovered"), + TransitionCandidateProbe::VersionedPresent("opaque-version".to_string()) + ); + assert_eq!( + backend + .probe_transition_candidate("unversioned-probe") + .await + .expect("unversioned candidate should be discovered"), + TransitionCandidateProbe::UnversionedPresent + ); + assert_eq!( + backend + .probe_transition_candidate("missing-probe") + .await + .expect("a missing key should be classified"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_candidate("provider-missing-probe") + .await + .expect("a provider-specific missing code should be classified"), + TransitionCandidateProbe::Missing + ); + let err = backend + .probe_transition_candidate("forbidden-probe") + .await + .expect_err("an authorization failure must not be mistaken for a missing key"); + assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied); + + let requests = fixture.await.expect("candidate fixture should join"); + for request in requests { + let request = request.to_ascii_lowercase(); + assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET"); + assert!(request.contains("\r\nrange: bytes=0-0\r\n")); + assert!(!request.contains("?versioning")); + assert!(!request.contains("?versions")); + } + } + fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { ListVersionsResult { versions: versions @@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 { } async fn probe_transition_candidate(&self, object: &str) -> Result { - let bucket_versioning = self.remote_bucket_versioning().await?; - self.probe_transition_candidate_versions(object, bucket_versioning).await + self.probe_current_transition_candidate_with_header(object, None).await } async fn in_use(&self) -> Result { diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index e2810e316..20eb7ee81 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::services::tier::{ tier_config::TierTencent, warm_backend::{ - S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options, + S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, new_s3_compatible_warm_backend, optimal_part_size, }, warm_backend_s3::WarmBackendS3, @@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent { self.0.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> Result { + self.0.probe_transition_candidate(object).await + } + async fn in_use(&self) -> Result { self.0.in_use().await } diff --git a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs index 2311e6576..8398aec52 100644 --- a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs +++ b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs @@ -23,7 +23,7 @@ use uuid::Uuid; use crate::services::tier::{ tier_config::{TierS3, TierWasabi}, - warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}; @@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi { self.s3.remove(object, rv).await } + async fn probe_transition_candidate(&self, object: &str) -> io::Result { + self.s3.probe_transition_candidate(object).await + } + async fn in_use(&self) -> io::Result { self.check_remote_bucket_unversioned().await?; let in_use = self.s3.in_use().await?; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index a89ef05f1..c88579c23 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -10339,9 +10339,17 @@ mod tests { } #[cfg(feature = "test-util")] - #[tokio::test] + #[test] #[serial_test::serial(storage_class_env)] - async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() { + fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() { + run_large_stack_async_test( + "v6-checkpoint-fence-loss", + v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case, + ); + } + + #[cfg(feature = "test-util")] + async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() { let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir"); let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await; diff --git a/crates/s3-client/src/api_error_response.rs b/crates/s3-client/src/api_error_response.rs index 8373517e7..7c653d6a8 100644 --- a/crates/s3-client/src/api_error_response.rs +++ b/crates/s3-client/src/api_error_response.rs @@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result where D: Deserializer<'de>, { - Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) + let code = String::deserialize(d)?; + if code == "NoSuchObject" { + return Ok(S3ErrorCode::NoSuchKey); + } + Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) } impl Default for ErrorResponse { @@ -325,4 +329,21 @@ mod tests { assert_eq!(response.code, S3ErrorCode::NoSuchVersion); assert_eq!(response.status_code, StatusCode::NOT_FOUND); } + + #[test] + fn normalizes_provider_specific_missing_object_code() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse")); + + let response = http_resp_to_error_response( + StatusCode::NOT_FOUND, + &headers, + b"NoSuchObjectremote detail".to_vec(), + "bucket", + "object", + ); + + assert_eq!(response.code, S3ErrorCode::NoSuchKey); + assert_eq!(response.status_code, StatusCode::NOT_FOUND); + } } diff --git a/crates/s3-client/src/api_get_object.rs b/crates/s3-client/src/api_get_object.rs index cc06fe388..872c1a90d 100644 --- a/crates/s3-client/src/api_get_object.rs +++ b/crates/s3-client/src/api_get_object.rs @@ -30,7 +30,9 @@ use tokio_util::io::StreamReader; use crate::{ api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, - transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider}, + transition_api::{ + ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider, + }, }; use futures_util::StreamExt; use http_body_util::BodyExt; @@ -39,6 +41,42 @@ use hyper::body::Bytes; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use tokio_util::io::ReaderStream; +fn response_limit_from_range(opts: &GetObjectOptions) -> Result, std::io::Error> { + let Some(range) = opts + .headers + .iter() + .find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str())) + else { + return Ok(None); + }; + let Some((unit, bounds)) = range.split_once('=') else { + return Ok(None); + }; + if !unit.eq_ignore_ascii_case("bytes") { + return Ok(None); + } + let Some((start, end)) = bounds.split_once('-') else { + return Ok(None); + }; + if start.is_empty() || end.is_empty() || end.contains(',') { + return Ok(None); + } + let start = start + .parse::() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?; + let end = end + .parse::() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?; + let length = end + .checked_sub(start) + .and_then(|length| length.checked_add(1)) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?; + let limit = usize::try_from(length).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory") + })?; + Ok(Some(limit)) +} + impl TransitionClient { pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result { let _ = opts; @@ -54,6 +92,7 @@ impl TransitionClient { object_name: &str, opts: &GetObjectOptions, ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let max_response_bytes = response_limit_from_range(opts)?; let resp = self .execute_method( http::Method::GET, @@ -81,18 +120,214 @@ impl TransitionClient { let h = resp.headers().clone(); - let mut body_vec = Vec::new(); let mut body = resp.into_body(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if let Some(data) = frame.data_ref() { - body_vec.extend_from_slice(data); + let body_vec = if let Some(limit) = max_response_bytes { + collect_response_body(body, limit).await? + } else { + let mut body_vec = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; + if let Some(data) = frame.data_ref() { + body_vec.extend_from_slice(data); + } } - } + body_vec + }; Ok((object_stat, h, BufReader::new(Cursor::new(body_vec)))) } } +#[cfg(test)] +mod bounded_response_tests { + use super::response_limit_from_range; + use crate::{ + api_get_options::GetObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body}, + }; + use http_body_util::Full; + use hyper::body::Bytes; + use std::time::Duration; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + #[test] + fn closed_range_derives_a_collection_limit_without_new_public_options() { + let mut opts = GetObjectOptions::default(); + opts.set_range(5, 11).expect("the closed range should be valid"); + + assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7)); + } + + #[tokio::test] + async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() { + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + let max_response_bytes = response_limit_from_range(&opts) + .expect("the range should parse") + .expect("the closed range should have a limit"); + let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes) + .await + .expect_err("the collection layer must reject a response larger than its limit"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle)> { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let request = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + loop { + let read = stream.read(&mut buffer).await.expect("fixture should read request headers"); + assert_ne!(read, 0, "connection closed before request headers were received"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request).into_owned(); + let response = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream + .write_all(response.as_bytes()) + .await + .expect("fixture should write response headers"); + stream.write_all(body).await.expect("fixture should write response body"); + request + }); + let client = TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "", + ) + .await + .expect("fixture client should build"); + Some((client, request)) + } + + #[tokio::test] + async fn real_transport_accepts_the_exact_closed_range_length() { + let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else { + return; + }; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let (_, _, mut reader) = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect("a seven-byte response should fit the requested range"); + let mut body = Vec::new(); + reader + .read_to_end(&mut body) + .await + .expect("bounded response should be readable"); + + assert_eq!(body, b"RustFS!"); + assert!( + request + .await + .expect("fixture should join") + .to_ascii_lowercase() + .contains("\r\nrange: bytes=0-6\r\n") + ); + } + + #[tokio::test] + async fn real_transport_rejects_a_body_larger_than_the_closed_range() { + let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else { + return; + }; + let mut opts = GetObjectOptions::default(); + opts.set_range(0, 6).expect("the probe range should be valid"); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("an eight-byte response must exceed the seven-byte range limit"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + request + .await + .expect("fixture should join") + .to_ascii_lowercase() + .contains("\r\nrange: bytes=0-6\r\n") + ); + } + + #[tokio::test] + async fn overflowing_closed_range_is_rejected_before_network_io() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("test listener should bind: {err}"), + }; + let endpoint = listener + .local_addr() + .expect("listener local address should be available") + .to_string(); + let client = TransitionClient::new( + &endpoint, + Options { + creds: Credentials::new(Static(Value { + access_key_id: "access-key".to_string(), + secret_access_key: "secret-key".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })), + region: "us-east-1".to_string(), + bucket_lookup: BucketLookupType::BucketLookupPath, + max_retries: 1, + ..Default::default() + }, + "", + ) + .await + .expect("fixture client should build"); + let mut opts = GetObjectOptions::default(); + opts.headers + .insert("range".to_string(), "bytes=0-18446744073709551615".to_string()); + + let err = client + .get_object_inner("bucket", "probe", &opts) + .await + .expect_err("an overflowing closed range must be rejected locally"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err() + ); + } +} + #[derive(Default)] pub struct GetRequest { pub buffer: Vec, diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 860ebb2cd..f568050d8 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -105,6 +105,12 @@ struct DirtyUsageSnapshot { covers_all_pending: bool, } +#[derive(Clone, Debug, Default)] +pub(crate) struct ScannerBucketScanScope { + selected_buckets: Option>>, + baseline_scan_plan_digest: Option, +} + pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR)) } @@ -146,6 +152,7 @@ fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool { pub struct ScannerBucketScanPlan { buckets: Vec, all_buckets: Arc>, + scope: ScannerBucketScanScope, digest: DataUsageScanPlanDigest, leader_epoch: u64, tier_registry_generation: u64, @@ -732,6 +739,8 @@ mod dirty_usage; mod guards; mod io_cache; mod io_cycle; +#[cfg(test)] +use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan}; pub(crate) use io_cycle::nsscanner_with_storage_status; mod io_disk; #[cfg(test)] diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index c99ffacd1..a8686a1e4 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -14,6 +14,93 @@ /// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish. use super::*; +#[derive(Clone, Copy)] +pub(super) struct ScannerSetCacheGeneration { + pub(super) want_cycle: u64, + pub(super) leader_epoch: u64, + pub(super) tier_registry_generation: u64, + pub(super) source: DataUsageCacheSource, + pub(super) scan_plan_digest: DataUsageScanPlanDigest, +} + +pub(super) struct PreparedScopedSetScan { + pub(super) buckets: Vec, + pub(super) cache: DataUsageCache, +} + +pub(super) fn prepare_scoped_set_scan( + old_cache: &DataUsageCache, + set_buckets: &[BucketInfo], + all_buckets: &[BucketInfo], + scope: &ScannerBucketScanScope, + generation: ScannerSetCacheGeneration, +) -> Option { + let (Some(selected_buckets), Some(baseline_scan_plan_digest)) = (&scope.selected_buckets, scope.baseline_scan_plan_digest) + else { + return None; + }; + if selected_buckets.is_empty() + || !old_cache.info.snapshot_complete + || old_cache.info.last_update.is_none() + || old_cache.info.name != DATA_USAGE_ROOT + || old_cache.info.next_cycle > generation.want_cycle + || old_cache.info.leader_epoch != generation.leader_epoch + || old_cache.info.tier_registry_generation != Some(generation.tier_registry_generation) + || old_cache.info.source != Some(generation.source) + || old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest) + || old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT + || old_cache.checked_flatten_complete_scope(DATA_USAGE_ROOT).is_none() + { + return None; + } + + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: generation.want_cycle, + leader_epoch: generation.leader_epoch, + tier_registry_generation: Some(generation.tier_registry_generation), + source: Some(generation.source), + snapshot_complete: false, + scan_plan_digest: Some(generation.scan_plan_digest), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + lkg_snapshot_complete: true, + lkg_next_cycle: Some(old_cache.info.next_cycle), + lkg_last_update: old_cache.info.last_update, + lkg_leader_epoch: Some(old_cache.info.leader_epoch), + lkg_scan_plan_digest: old_cache.info.scan_plan_digest, + ..Default::default() + }, + cache: HashMap::new(), + }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + let root_hash = crate::hash_path(DATA_USAGE_ROOT); + let mut current_bucket_names = HashSet::with_capacity(all_buckets.len()); + for bucket in all_buckets { + if !current_bucket_names.insert(bucket.name.as_str()) { + return None; + } + if selected_buckets.contains(&bucket.name) { + cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + continue; + } + + let bucket_hash = crate::hash_path(&bucket.name); + old_cache.find(&bucket.name)?; + cache.copy_with_children(old_cache, &bucket_hash, &Some(root_hash.clone())); + cache.find(&bucket.name)?; + } + + Some(PreparedScopedSetScan { + buckets: set_buckets + .iter() + .filter(|bucket| selected_buckets.contains(&bucket.name)) + .cloned() + .collect(), + cache, + }) +} + #[async_trait::async_trait] impl ScannerIOCache for SetDisks { #[tracing::instrument(skip(self, budget, scan_plan, updates))] @@ -27,8 +114,9 @@ impl ScannerIOCache for SetDisks { scan_mode: HealScanMode, ) -> Result<()> { let ScannerBucketScanPlan { - buckets, + mut buckets, all_buckets, + scope, digest: scan_plan_digest, leader_epoch, tier_registry_generation, @@ -63,26 +151,57 @@ impl ScannerIOCache for SetDisks { "Scanner old data usage cache load failed; rebuilding from bucket caches" ); } + let scoped_scan = prepare_scoped_set_scan( + &old_cache, + &buckets, + &all_buckets, + &scope, + ScannerSetCacheGeneration { + want_cycle, + leader_epoch, + tier_registry_generation, + source, + scan_plan_digest, + }, + ); + let mut scoped_cache = scoped_scan.map(|prepared| { + buckets = prepared.buckets; + prepared.cache + }); if buckets.is_empty() { let now = SystemTime::now(); - let mut cache = DataUsageCache { - info: DataUsageCacheInfo { - name: DATA_USAGE_ROOT.to_string(), - next_cycle: want_cycle, - last_update: Some(now), - leader_epoch, - tier_registry_generation: Some(tier_registry_generation), - source: Some(source), - snapshot_complete: true, - scan_plan_digest: Some(scan_plan_digest), - cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, - ..Default::default() - }, - cache: HashMap::new(), + let mut cache = match scoped_cache.take() { + Some(cache) => cache, + None => { + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: want_cycle, + leader_epoch, + tier_registry_generation: Some(tier_registry_generation), + source: Some(source), + scan_plan_digest: Some(scan_plan_digest), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + ..Default::default() + }, + cache: HashMap::new(), + }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + for bucket in all_buckets.iter() { + cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + } + cache + } }; - cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); - for bucket in all_buckets.iter() { - cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + cache.info.last_update = Some(now); + cache.info.snapshot_complete = true; + cache.info.lkg_snapshot_complete = false; + cache.info.lkg_next_cycle = None; + cache.info.lkg_last_update = None; + cache.info.lkg_leader_epoch = None; + cache.info.lkg_scan_plan_digest = None; + if cache.find(DATA_USAGE_ROOT).is_none() { + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); } reset_disk_bucket_scan_gauges(&pool_label, &set_label); return persist_and_publish_cache_snapshot( @@ -269,92 +388,102 @@ impl ScannerIOCache for SetDisks { record_disk_bucket_scans_active(0, &pool_label, &set_label); let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone()); - // Fence a stale set aggregate before copying entries into per-bucket work caches. - if old_cache.info.next_cycle <= want_cycle - && old_cache.info.leader_epoch <= leader_epoch - && old_cache.info.tier_registry_generation != Some(tier_registry_generation) - { - old_cache.info.scan_plan_digest = None; - } - let old_lkg = old_cache.info.snapshot_complete.then_some({ - ( - old_cache.info.next_cycle, - old_cache.info.last_update, - old_cache.info.leader_epoch, - old_cache.info.scan_plan_digest, - ) - }); - let prepare_outcome = match old_cache.prepare_for_scan( - DATA_USAGE_ROOT, - want_cycle, - leader_epoch, - source, - scan_plan_digest, - require_cache_source, - ) { - DataUsageCachePrepareOutcome::RejectedNewerCycle => { - cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel); - warn!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - pool = self.pool_index, - set = self.set_index, - cache_name = DATA_USAGE_CACHE_NAME, - requested_cycle = want_cycle, - cached_cycle = old_cache.info.next_cycle, - state = "stale_cycle_rejected", - "Scanner rejected a set cache cycle regression" - ); - return Ok(()); + let mut cache = if let Some(cache) = scoped_cache.take() { + cache + } else { + // Fence a stale set aggregate before copying entries into per-bucket work caches. + if old_cache.info.next_cycle <= want_cycle + && old_cache.info.leader_epoch <= leader_epoch + && old_cache.info.tier_registry_generation != Some(tier_registry_generation) + { + old_cache.info.scan_plan_digest = None; } - DataUsageCachePrepareOutcome::RejectedNewerLeader => { - warn!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - pool = self.pool_index, - set = self.set_index, - cache_name = DATA_USAGE_CACHE_NAME, - requested_epoch = leader_epoch, - cached_epoch = old_cache.info.leader_epoch, - state = "stale_leader_rejected", - "Scanner rejected work from an older leader epoch" - ); - return Ok(()); - } - outcome => outcome, - }; - if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused) - && let Some((cycle, last_update, epoch, digest)) = old_lkg - { - old_cache.info.lkg_snapshot_complete = true; - old_cache.info.lkg_next_cycle = Some(cycle); - old_cache.info.lkg_last_update = last_update; - old_cache.info.lkg_leader_epoch = Some(epoch); - old_cache.info.lkg_scan_plan_digest = digest; - } - - let mut cache = DataUsageCache { - info: DataUsageCacheInfo { - name: DATA_USAGE_ROOT.to_string(), - next_cycle: want_cycle, + let old_lkg = old_cache.info.snapshot_complete.then_some({ + ( + old_cache.info.next_cycle, + old_cache.info.last_update, + old_cache.info.leader_epoch, + old_cache.info.scan_plan_digest, + ) + }); + let prepare_outcome = match old_cache.prepare_for_scan( + DATA_USAGE_ROOT, + want_cycle, leader_epoch, - tier_registry_generation: Some(tier_registry_generation), - source: Some(source), - snapshot_complete: false, - scan_plan_digest: Some(scan_plan_digest), - cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, - ..Default::default() - }, - cache: HashMap::new(), + source, + scan_plan_digest, + require_cache_source, + ) { + DataUsageCachePrepareOutcome::RejectedNewerCycle => { + cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel); + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + requested_cycle = want_cycle, + cached_cycle = old_cache.info.next_cycle, + state = "stale_cycle_rejected", + "Scanner rejected a set cache cycle regression" + ); + return Ok(()); + } + DataUsageCachePrepareOutcome::RejectedNewerLeader => { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + requested_epoch = leader_epoch, + cached_epoch = old_cache.info.leader_epoch, + state = "stale_leader_rejected", + "Scanner rejected work from an older leader epoch" + ); + return Ok(()); + } + outcome => outcome, + }; + if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused) + && let Some((cycle, last_update, epoch, digest)) = old_lkg + { + old_cache.info.lkg_snapshot_complete = true; + old_cache.info.lkg_next_cycle = Some(cycle); + old_cache.info.lkg_last_update = last_update; + old_cache.info.lkg_leader_epoch = Some(epoch); + old_cache.info.lkg_scan_plan_digest = digest; + } + + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: want_cycle, + leader_epoch, + tier_registry_generation: Some(tier_registry_generation), + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(scan_plan_digest), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + lkg_snapshot_complete: old_cache.info.lkg_snapshot_complete, + lkg_next_cycle: old_cache.info.lkg_next_cycle, + lkg_last_update: old_cache.info.lkg_last_update, + lkg_leader_epoch: old_cache.info.lkg_leader_epoch, + lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest, + ..Default::default() + }, + cache: HashMap::new(), + }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + for bucket in all_buckets.iter() { + cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); + } + cache }; - cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); - for bucket in all_buckets.iter() { - cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); - } let (bucket_tx, bucket_rx) = mpsc::channel::(buckets.len()); @@ -1257,11 +1386,6 @@ impl ScannerIOCache for SetDisks { incomplete_scope.info.snapshot_complete = false; incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; - incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete; - incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle; - incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update; - incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch; - incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest; if let Err(e) = updates.send(incomplete_scope).await { error!( target: "rustfs::scanner::io", diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index f51caf381..3b14e9ff2 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -63,6 +63,41 @@ pub(crate) async fn nsscanner_with_storage_status( where S: ScannerStorage, { + let request = ScannerCycleRequest { + ctx, + budget, + updates, + want_cycle, + leader_epoch, + scan_mode, + scan_scope: ScannerBucketScanScope::default(), + }; + nsscanner_with_storage_status_scoped(store, request).await +} + +pub(crate) struct ScannerCycleRequest { + pub(crate) ctx: CancellationToken, + pub(crate) budget: Arc, + pub(crate) updates: mpsc::Sender, + pub(crate) want_cycle: u64, + pub(crate) leader_epoch: u64, + pub(crate) scan_mode: HealScanMode, + pub(crate) scan_scope: ScannerBucketScanScope, +} + +pub(crate) async fn nsscanner_with_storage_status_scoped(store: &S, request: ScannerCycleRequest) -> Result +where + S: ScannerStorage, +{ + let ScannerCycleRequest { + ctx, + budget, + updates, + want_cycle, + leader_epoch, + scan_mode, + scan_scope, + } = request; let child_token = ctx.child_token(); let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); @@ -280,6 +315,7 @@ where let scan_plan = ScannerBucketScanPlan { buckets: set_buckets, all_buckets: Arc::clone(&all_buckets), + scope: scan_scope.clone(), digest: scan_plan_digest, leader_epoch, tier_registry_generation, diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 47f1e86a0..a59d620fc 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -765,6 +765,155 @@ fn bucket_usage_scan_order_prioritizes_dirty_buckets() { assert_eq!(names, vec!["dirty", "missing", "cached"]); } +fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsageScanPlanDigest) -> DataUsageCache { + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: 7, + last_update: Some(SystemTime::now()), + leader_epoch: 11, + source: Some(DataUsageCacheSource::new(1, 2)), + snapshot_complete: true, + scan_plan_digest: Some(scan_plan_digest), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + tier_registry_generation: Some(13), + ..Default::default() + }, + ..Default::default() + }; + cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); + for (bucket, size) in buckets { + cache.replace( + bucket, + DATA_USAGE_ROOT, + DataUsageEntry { + size: *size, + objects: 1, + ..Default::default() + }, + ); + } + cache +} + +#[test] +fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() { + let baseline_digest = DataUsageScanPlanDigest([1; 32]); + let current_digest = DataUsageScanPlanDigest([2; 32]); + let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20), ("deleted", 30)], baseline_digest); + old_cache.replace( + "stable/prefix", + "stable", + DataUsageEntry { + size: 5, + objects: 1, + ..Default::default() + }, + ); + let all_buckets = vec![bucket_info("stable"), bucket_info("dirty")]; + let selected_buckets = Arc::new(HashSet::from(["dirty".to_string(), "deleted".to_string()])); + + let prepared = prepare_scoped_set_scan( + &old_cache, + &all_buckets, + &all_buckets, + &ScannerBucketScanScope { + selected_buckets: Some(selected_buckets), + baseline_scan_plan_digest: Some(baseline_digest), + }, + ScannerSetCacheGeneration { + want_cycle: 8, + leader_epoch: 11, + tier_registry_generation: 13, + source: DataUsageCacheSource::new(1, 2), + scan_plan_digest: current_digest, + }, + ) + .expect("complete matching set cache should support a scoped scan"); + + assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::>(), ["dirty"]); + let stable = prepared + .cache + .checked_flatten("stable") + .expect("unselected bucket subtree should be retained"); + assert_eq!((stable.size, stable.objects), (15, 2)); + assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0))); + assert!(prepared.cache.find("deleted").is_none()); + assert_eq!(prepared.cache.info.scan_plan_digest, Some(current_digest)); + assert_eq!(prepared.cache.info.next_cycle, 8); + assert!(!prepared.cache.info.snapshot_complete); + assert!(prepared.cache.info.lkg_snapshot_complete); + assert_eq!(prepared.cache.info.lkg_next_cycle, Some(7)); + assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest)); +} + +#[test] +fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() { + let baseline_digest = DataUsageScanPlanDigest([3; 32]); + let old_cache = complete_set_usage_cache(&[("stable", 10)], baseline_digest); + let all_buckets = vec![bucket_info("stable"), bucket_info("new")]; + + assert!( + prepare_scoped_set_scan( + &old_cache, + &all_buckets, + &all_buckets, + &ScannerBucketScanScope { + selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))), + baseline_scan_plan_digest: Some(baseline_digest), + }, + ScannerSetCacheGeneration { + want_cycle: 8, + leader_epoch: 11, + tier_registry_generation: 13, + source: DataUsageCacheSource::new(1, 2), + scan_plan_digest: DataUsageScanPlanDigest([4; 32]), + }, + ) + .is_none() + ); +} + +#[test] +fn scoped_set_scan_requires_an_exact_complete_baseline() { + let baseline_digest = DataUsageScanPlanDigest([5; 32]); + let all_buckets = vec![bucket_info("dirty")]; + let scope = ScannerBucketScanScope { + selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))), + baseline_scan_plan_digest: Some(baseline_digest), + }; + let generation = ScannerSetCacheGeneration { + want_cycle: 8, + leader_epoch: 11, + tier_registry_generation: 13, + source: DataUsageCacheSource::new(1, 2), + scan_plan_digest: DataUsageScanPlanDigest([6; 32]), + }; + + let mut incomplete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest); + incomplete.info.snapshot_complete = false; + assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation).is_none()); + + let mut not_durable = complete_set_usage_cache(&[("dirty", 10)], baseline_digest); + not_durable.info.last_update = None; + assert!(prepare_scoped_set_scan(¬_durable, &all_buckets, &all_buckets, &scope, generation).is_none()); + + let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest); + wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32])); + assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none()); + + let empty_scope = ScannerBucketScanScope { + selected_buckets: Some(Arc::new(HashSet::new())), + baseline_scan_plan_digest: Some(baseline_digest), + }; + let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest); + assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none()); + + let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest); + future_cache.info.next_cycle = generation.want_cycle.saturating_add(1); + assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation).is_none()); +} + #[test] fn record_set_scan_failure_preserves_first_error() { let mut first = None; diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index ccf3d9995..750ec2f43 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -41,7 +41,7 @@ All keys below are objects in the internal metadata bucket. The table gives the | Protocol | Current schema/version | Canonical key | Creator and cleanup owner | Authoritative identity and mutable fields | Current durability point | |---|---|---|---|---|---| -| Transition transaction | `rustfs-transition-transaction-v1` | `ilm/transition-transactions/records///.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Maximum-parity config write. Current create/update/delete calls do not use ETag preconditions | +| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records///.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete | | Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records///.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity | | Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators///.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent | | Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs///.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS | @@ -54,6 +54,7 @@ All keys below are objects in the internal metadata bucket. The table gives the | Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion | | Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts////.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity | | Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal | +| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup | `durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object. @@ -71,7 +72,7 @@ The internal config layer supplies maximum-parity writes, create-only writes, ET | Conditional delete | Removes only the verified terminal generation; if an active decommission covers it, its terminal receipt is written first | | Strong readback | Resolves a lost response only when key, schema, full immutable identity, state, and expected successor all match | -Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. The transition transaction currently carries a fixed `owner_epoch` fence identity and a mutable `revision`, but persists with unconditional writes and deletes; those fields therefore detect some in-memory misuse but are not yet a durable exclusion fence. Changing `owner_epoch` during takeover is not current behavior and remains an open design. The manual job's initial UUID-key write has the same create-only gap, although all later owner/lease updates are CAS-protected. +Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. Transition transaction v1 now uses create-only installation, exact record plus ETag read before each successor CAS, and exact ETag terminal deletion. Its `owner_epoch` and `not_after_unix_nanos` remain immutable, however, so an expired recovery worker claims only the next state generation rather than a renewable durable owner lease. The manual job's initial UUID-key write still lacks create-only installation, although all later owner/lease updates are CAS-protected. ### Approved target @@ -81,7 +82,7 @@ Tier mutation intents and v6 journal/manifest records use the conditional primit - After a timeout, connection loss, or quorum-uncertain response, the caller must strongly reread. Only the exact intended successor is success; predecessor, absence, conflict, corruption, or unavailable readback retains the record and blocks destructive action. - A process-local mutex, cancellation token, task registry, or cached generation may reduce duplicate work but cannot authorize publication, rollback, or remote deletion. -The exact transition-transaction lease/takeover fields and whether the existing `not_after` becomes the owner expiry are an **open design**. They must be settled with upgrade/downgrade behavior before the v1 schema changes. +The approved transition-transaction successor, lease/takeover fields, v1 migration, and upgrade/downgrade gates are specified in [Bounded recovery control and operator disposition](#bounded-recovery-control-and-operator-disposition). They require implementation and fleet gating before any v2 writer or destructive v1 takeover is enabled. ## Lock and operation order @@ -89,18 +90,19 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n | Path | Current acquisition order | Operations allowed while held | Operations forbidden while held | |---|---|---|---| -| Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation | +| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. Both guards are released for lease drain, peer Prepare, and reference proof, then reacquired in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, peer fanout, and reference proof run without either exclusive guard. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation | | v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition | | v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery | | v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version | | v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version | | Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes | | Transition commit | Existing object commit locks plus exact source identity and tier-generation checks in the transition path | Publish the exact remote tuple into the matching local version | Publishing a tuple after the source identity or generation changes | -| Transition transaction cleanup | Record validation; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit bucket-lifecycle/physical-set ownership fence or durable takeover CAS | Identity-bound provider probe and deletion of a known canonical candidate | A tier lease alone does not fence the creator. The approved target requires expired ownership, durable takeover, and exact local/source reread before DELETE | +| Transition transaction cleanup | Record validation; expiry check; a next-state ETag CAS for cleanup ownership; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit renewable owner lease or bucket-lifecycle/physical-set ownership fence | Identity-bound provider probe and deletion of a known canonical candidate | The cleanup-state CAS fences a stale predecessor, but the approved target also requires an explicit recovery lease and exact all-pool source reread before DELETE | +| Recovery artifact quota admission (approved target) | Cluster-scoped recovery-admission WRITE lock first; then the one canonical control/source operation lock and any source-protocol metadata/physical locks in that protocol's existing stable order | Bounded internal artifact inventory, exact source/control validation, and create-only installation plus strong readback of one fully encoded export or disposition candidate | Acquiring admission while holding a control, source, disposition, bucket, physical, migration, or decommission guard; remote backend I/O; source-journal deletion; disposition `Applying`; releasing admission before candidate installation/readback converges | | Manual job | Initial maximum-parity job write; then persisted bucket-level scope create/CAS; later short job/task/result metadata operations | List, checkpoint, append tasks before enqueue, append results after work, renew/take over lease | Holding metadata guards across remote transition PUT; treating the local active-job map as cluster authority. A crash between the job write and scope claim can leave `Running` without a scope record | | Decommission receipt | Decommission coordinator's source/target record workflow; record-specific conditional writes | Copy/validate durable record, advance receipts, construct and verify expected manifest, conditionally clean exact covered source | Remote tier DELETE; deleting an uncovered or divergent source record | -The tier mutation lock scope is intentionally recorded as **current**, not ideal. Reducing it is allowed only after a durable `Prepared` intent blocks new reference creators across the fleet, existing tier-operation leases drain, and recovery can reconstruct that block without the initiating process. Which network validation can move outside the namespace lock is an **open design**. +Tier mutation backend validation is outside both exclusive guards and is bound to the persisted ETag snapshot. The initiating task is detached from the admin request so cancellation cannot interrupt validation cleanup. Once the coordinator Prepared record and local fence are durable, lease drain, all-node peer Prepare, and reference proof run without the tier-config namespace or `admin_updates` guard. Recovery treats an unexpired Prepared record as active even while the old config ETag remains current. Both guards are reacquired in namespace-then-admin order, and the ETag, candidate digest, exact intent identity, and expiry are revalidated immediately before config CAS. After the coordinator advances to Committed, the guards are released again for peer Commit and local publication. ## Transition transaction @@ -114,7 +116,7 @@ UploadStarted -> Uploaded -> LocalCommitStarted -> Committed \-> AbortedNoRemote ``` -Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production code emits `CleanupPending` only when recovery probes `UploadOutcomeUnknown` as `UnversionedPresent` or as `VersionedPresent` with a non-nil identifier. The `Uploaded` abort/recovery path deletes its candidate and transaction record directly, and `LocalCommitStarted` mismatch or missing-source recovery retains the record. The `Uploaded` and `LocalCommitStarted` cleanup edges are currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is another current special case: recovery exact-deletes it and removes the record without first persisting `CleanupPending`. +Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production recovery emits `CleanupPending` after an expired `Uploaded` record wins the exact successor CAS, or when an expired `UploadOutcomeUnknown` probe returns `UnversionedPresent` or `VersionedPresent` with a non-nil identifier. `LocalCommitStarted` mismatch or missing-source recovery retains the record; that cleanup edge is currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is retained and never authorizes remote deletion. The remote candidate itself is named by `canonical_transition_remote_object` under `ilm/transition-transactions////`. That deterministic identity is what a provider probe or exact cleanup must bind; it is distinct from the internal transaction-record key. @@ -124,13 +126,13 @@ The creator owns the canonical remote candidate until local metadata commits the | Observed durable state/input | Unique current owner | Current recovery decision | Approved destructive admission | |---|---|---|---| -| `UploadStarted` | Originating transition attempt; current durable exclusion is incomplete | Retain | No delete. The upload may still publish | -| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery, logically; current record writes do not durably exclude a concurrent worker | Delete the record | Strong probe identity must match transaction/backend; no remote delete occurs | +| `UploadStarted` | Originating transition attempt; no current recovery successor is emitted | Retain | No delete. The upload may still publish | +| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery under the exact record generation and recovery lock | Conditionally delete the record | Strong probe identity must match transaction/backend; no remote delete occurs | | `UploadOutcomeUnknown`; probe returns `UnversionedPresent` | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, delete the unversioned candidate, delete the record | Exact transaction/canonical object/backend identity, explicitly unversioned state, durable takeover after owner expiry, current tier lease, and exact reread before cleanup | | `UploadOutcomeUnknown`; probe returns `VersionedPresent` with a non-nil exact identifier | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, exact-delete that versioned candidate, delete the record | Exact transaction/canonical object/backend identity and remote version, durable takeover after owner expiry, current tier lease, and exact reread before cleanup | -| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery | Current code directly exact-deletes that versioned candidate and deletes the record; it does not persist `CleanupPending` | This remains a versioned exact-delete candidate and must not be treated as `UnversionedPresent`. The approved target still requires durable takeover, a current tier lease, and exact reread | +| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery retains ownership evidence | Retain | A nil identifier is invalid exact-version evidence and never becomes unversioned or remote-delete authority | | `UploadOutcomeUnknown`; probe ambiguous, unsupported, or errors | Transaction recovery retains ownership evidence | Retain | No destructive action; operator reconcile may inspect after expiry | -| `Uploaded` | Originating transition attempt; current recovery can race it because the persisted fence is not CAS-protected | Current code immediately deletes the candidate and record | **Current safety gap:** approved behavior must first prove the creator cannot still commit by expired ownership plus durable takeover/CAS, then recheck that no matching local commit exists | +| `Uploaded` | Originating transition attempt until expiry; after expiry, the worker that wins `Uploaded -> CleanupPending` by exact ETag CAS | Retain while active; after expiry, persist `CleanupPending`, then recheck and delete the unreferenced candidate or record | Current CAS fences the predecessor, but approved v2 also requires a durable recovery lease, full all-pool source/free-version proof, and before/after fence checks | | `LocalCommitStarted`; logical source lookup returns `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Current recovery treats this tuple as ownership transfer. The approved target additionally compares recorded source version ID, data directory, modification time, size, and ETag before conditional terminal cleanup | | `LocalCommitStarted`; logical source is missing, its transition tuple differs, or the read is uncertain | Transaction record/recovery | Retain | No remote delete without a separate durable cleanup proof | | `CleanupPending`; logical source lookup returns the same current transition predicate | `xl.meta` is remote reachability owner; recovery owns only record cleanup | Delete transaction record | `xl.meta` is owner; do not delete remote. The approved target adds the full recorded source comparison | @@ -148,12 +150,12 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p ### Approved target and open design -- Replace unconditional transaction create/update/delete with create-only, ETag CAS, and conditional terminal delete. +- Preserve the current create-only transaction installation, exact record/ETag successor CAS, and conditional terminal delete, and add mandatory exact-successor strong readback for lost or uncertain responses. No v2 work may regress the existing v1 protections. - Recovery of `Uploaded` and cleanup-capable states must acquire durable ownership only after the prior owner's expiry. The recovery worker must reread the exact generation after takeover and before remote DELETE. - Preserve and compare source version ID, data directory, modification time, size, and ETag through local commit and recovery before accepting ownership transfer. - Recompute and require the exact lowercase sharded path in both transition-transaction and manual-job runtime recovery, and validate a truncated page's continuation token before processing any record from that page. -- **Open:** owner lease duration, clock-skew allowance, takeover revision encoding, and compatibility for existing v1 records that have only `not_after`/`owner_epoch`. -- **Open:** bounded retention and an operator disposition for permanently ambiguous records. Until defined, retained ambiguity is safer than collection. +- The approved v2 owner lease uses the duration, clock-skew allowance, takeover revision, and v1 mapping in [Transition transaction v2 and v1 migration](#transition-transaction-v2-and-v1-migration). +- Active automatic retries are bounded by the recovery-control policy below. Ambiguous evidence becomes explicit `retained_ambiguous` or `operator_required`; source evidence is never collected merely because it is old. ## Tier mutation intent @@ -163,14 +165,15 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence. -The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. The coordinator then conditionally writes tier config, commits peers, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative. +The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative. ### Recovery decisions | Observed durable state/input | Unique current owner | Current recovery decision | Destructive/config admission | |---|---|---|---| | `Prepared`; current tier-config digest equals candidate | Coordinator recovery; each peer recovery owns only its matching peer record/block | CAS to `Committed`, replay peer Commit and publish | Exact mutation identity and candidate digest; never infer from expiry | -| `Prepared`; current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone | +| `Prepared`; intent is unexpired and current config still proves the old ETag/config | The live coordinator remains owner | Retain `Prepared` and the runtime block | Recovery must not abort work that may still be in lock-free peer Prepare or reference proof | +| `Prepared`; intent is expired and current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone | | `Prepared`; config is a third generation, unreadable, or peer outcome is ambiguous | Coordinator record remains owner of the block | Retain `Prepared` and runtime block | No commit, abort, cleanup, or unblock | | `Committed` | Coordinator recovery, with peers owning convergence of their local records | Replay peer Commit/runtime publication; clean exact converged records | Config ETag/digest and peer identity must match | | `Aborted` | Coordinator recovery; peer recovery retains the local tombstone | Replay/confirm Abort and clear matching block; retain peer tombstone until expiry plus clock skew and no coordinator | Never roll back config based on timeout alone | @@ -183,8 +186,8 @@ Intent transitions retry an ETag race at most three times before returning a ret ### Approved target and open design - Keep create-only, ETag CAS, exact identity comparison, canonical Abort tombstones, and lost-response readback. -- Any shorter configuration-lock window must leave a durable `Prepared` fence installed on every required peer before releasing the broad exclusion scope and must prove recovery restores that fence before admitting reference creators. -- **Open:** the exact split between backend validation, peer fanout, reference scan, config CAS, and publication; expiry must never replace config-generation proof. +- The phase split is fixed: ETag snapshot, lock-free backend validation, ETag revalidation, all-node Prepare, reference proof, ETag/digest/intent revalidation, config CAS, all-node Commit, and local publication. Backend validation uses `(old_config_etag, candidate_digest)` as its stable config generation; the durable mutation identity adds `mutation_id`. Runtime driver revisions are not persistence identities and Add does not require one before publication. +- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof. - **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records. ## Manual transition job, task, result, and checkpoint @@ -311,7 +314,149 @@ Single and child manifest preparation is bounded by 200,000 journals and a 32 Mi - New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created. - Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE. - Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous. -- **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof. +- Quarantined v1/v2 records, incomplete manifests, and repeatedly failing exact deletes use the bounded retry, explicit operator state, and single-record control surface approved below. Capacity and recovery throughput must not be “fixed” by weakening ownership proof or age-deleting source evidence. + +## Bounded recovery control and operator disposition + +This section is an **approved target that is not implemented yet**. It closes the ownership, retry, and operator-disposition design required before backlog recovery work can change destructive behavior. It does not authorize an implementation to enable a v2 writer, take over a v1 transaction, or remove a legacy journal until the fleet and storage gates below exist. + +### Recovery-control record + +Retry scheduling is persisted separately from the source transaction or journal so legacy bytes remain readable and their cleanup ownership does not move. The control record uses schema `rustfs-ilm-recovery-control-v1`, a checksum envelope, a 16 KiB encoded-size limit, strict unknown-field rejection, and this canonical key: + +```text +ilm/recovery-controls////.json +``` + +Export envelopes use `ilm/recovery-exports////.json`; disposition receipts use `ilm/recovery-dispositions////.json`. `export-id` is the SHA-256 of the control ID plus exact observed source content/copy-set digests; `disposition-id` is the SHA-256 of the export ID plus the bounded action. Repeating an identical export or disposition reuses and strongly validates the same canonical object; different bytes at that ID are a conflict. Both use strict checksum envelopes and create-only installation. A control or disposition is limited to 16 KiB. An export is limited to the source protocol's own maximum encoded record size plus 64 KiB for its copy manifest and envelope; it stores the source bytes once rather than duplicating identical replica bytes. + +Admission is fail-closed and cluster-wide. One control may have at most one export creation and one disposition application in flight. The immutable candidate is fully encoded before quota admission. The cluster-scoped recovery-admission WRITE lock is always outermost; a caller already holding any control, source, disposition, bucket, physical, migration, or decommission guard must release it and restart in the order above. While admission is held, a complete artifact inventory must prove that the projected totals, including the candidate, are at most 10,000 export envelopes, 10,000 disposition receipts, 1 GiB of encoded export data, and 256 MiB of encoded control/disposition data. The create-only candidate installation and exact strong readback complete before the lock is released, so neither a single candidate nor a concurrent node can oversubscribe the pre-create snapshot. A crash before installation leaves no artifact; a lost create response is resolved by exact canonical readback while admission remains serialized or after reacquiring it in the same order. Replaying an existing canonical ID consumes no new count, byte, or rate token. New creations are limited to ten per authenticated actor per minute and 100 cluster-wide per minute, with at most 32 export creations and eight disposition applications executing cluster-wide. Exceeding a count, byte, rate, or concurrency limit returns a retryable capacity result before source mutation; it never evicts evidence, interrupts an admitted operation, or blocks ordinary object I/O. The collector examines at most 100 terminal artifacts per minute and never acquires the admission lock while holding an artifact/source guard. + +`source-operation-digest` is SHA-256 over the length-delimited source protocol, canonical source path, and stable semantic operation identity: transaction UUID, journal identity, or manifest operation ID. For corrupt bytes whose semantic ID cannot be trusted, the canonical path plus a `corrupt` domain separator is the stable identity; a replacement at that path conflicts with the existing control instead of resetting its history. The control's immutable identity repeats the protocol, path, stable identity, and record class. Its mutable `observed_source_generation` contains the source schema, source ETag/content SHA-256, and sorted all-pool copy-set digest. The copy set records each authoritative pool/set identity, canonical path, ETag, byte length, and content SHA-256; an unreachable pool, missing ETag, divergent copy, or incomplete listing cannot produce an actionable generation. The remaining mutable generation contains: + +- `revision`, an ETag-CAS successor counter; +- `classification`: `retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`; +- `owner_id`, `owner_epoch`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos` while an attempt is owned; +- monotonic `attempt_count`, `consecutive_failure_count`, `first_failure_at_unix_nanos`, `last_failure_at_unix_nanos`, and `next_attempt_at_unix_nanos`; +- one bounded enum `last_error_code`; no free-form provider error, endpoint, credential material, request payload, or response body is persisted; +- for an operator disposition only, the authenticated actor identifier, reason code, confirmation time, exported payload digest, and exact source/control ETags that were confirmed. + +The control record is a scheduler and audit fence, not a remote-object owner. It never substitutes for the source record's version, backend, manifest, source-identity, or absence proof. Before every source CAS, local cleanup, or remote request, the worker strongly rereads every authoritative source copy and the control, requires the current observed generation to match, and then applies the source protocol's own locks and proof. A missing, stale, corrupt, divergent, or unavailable control read cannot authorize work. + +Control creation is `If-None-Match: *`; every update is exact ETag `If-Match`; response loss requires exact strong readback. A legal source-state CAS does not create a new control. The stable control CAS advances `observed_source_generation` only from the exact predecessor to one source-protocol successor while preserving `first_failure_at_unix_nanos`, lifetime `attempt_count`, and first-seen lineage. If the source CAS succeeded before a crash, recovery accepts the new bytes only after validating that exact legal successor and then converges the old control generation by CAS. A multi-edge jump, semantic-identity change, replacement ETag/content, or unavailable predecessor proof is a conflict and cannot reset counters. Recovery conditionally removes a terminal control only after strongly proving the stable source operation resolved or absent, recording any active decommission terminal receipt, and confirming that no in-flight operator request still names its ETag. `durable_namespace.rs` must register every new namespace, path parser, decoder, size bound, successor relation, and terminal checkpoint before rollout. + +### Transition transaction v2 and v1 migration + +The successor envelope is `rustfs-transition-transaction-v2`. It preserves the v1 transaction ID, deployment ID, write ID, complete source identity, tier/backend identity, canonical remote object, remote-version state, operation state, and revision. It also records an immutable `origin_format` (`native_v2` or `migrated_v1`), the state-entry revision and predecessor state/revision, and, for migration, the exact source v1 state/revision. These fields make the state history needed for destructive authorization independently checkable instead of inferring it from the current state. The envelope and body reject unknown fields, checksum every field, use no default for a required v2 value, require the existing exact lowercase canonical path, and reject nil UUIDs, nonpositive timestamps, revision zero/overflow, lease inversion, impossible state/revision history, and illegal state/version combinations. It replaces the fixed ownership pair with: + +- immutable `creator_epoch` and `creator_not_after_unix_nanos`, copied exactly from v1 `owner_epoch` and `not_after_unix_nanos` during migration; +- optional `created_at_unix_nanos`; migrated v1 records use `None` rather than inventing an age; +- mutable `owner_role` (`creator` or `recovery`), `owner_id`, `owner_epoch`, `owner_generation`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos`. `owner_id` is the authenticated stable fleet-node identity; `owner_epoch` is a fresh UUID for one process claim. A node without both identities cannot create, renew, take over, or act on v2. + +A native v2 create uses revision and owner generation 1, a fresh non-nil creator owner/epoch, `UploadStarted`, and an unknown remote version. The first rollout keeps the current seven-day creator ownership window. Creator leases are not renewable; a creator that cannot finish inside the safety window stops publishing and leaves the record for recovery. Recovery-owner leases are 15 minutes. The persisted clock-skew allowance is five minutes: an action may start only when its bounded deadline fits before `lease_expires_at - 5 minutes`, and another owner cannot take over until its local time is at least `lease_expires_at + 5 minutes`. A recovery attempt remains capped at five minutes and every remote call remains subject to its narrower client deadline. A recovery renewal is a same-owner, next-revision CAS that strictly extends expiry; a takeover changes `owner_role`, `owner_id`, and `owner_epoch`, increments `owner_generation` and `revision`, and uses the exact observed ETag. Takeover and state advancement are separate CAS operations; one revision cannot both acquire ownership and claim a recovery outcome. + +Before migration can be enabled, the fleet must first deploy a v1 creator fence that strongly rereads the exact transaction path, ETag, state, owner epoch, and source generation immediately before local metadata publication and refuses to publish after any migration/takeover change. The fleet then durably disables new v1 admission and proves that every captured creator/recovery process epoch has either acknowledged quiescence or terminated. A paused or unreachable epoch prevents migration. This drain barrier is distinct from format capability advertisement and remains in force until v2 writer admission is enabled. + +Only these checksum-valid v1 state/revision pairs are migration inputs: `UploadStarted@1`; `UploadOutcomeUnknown@2`; `AbortedNoRemote@2`; `Uploaded@2` or `Uploaded@3`; `LocalCommitStarted@3` or `LocalCommitStarted@4`; `Committed@4` or `Committed@5`; and `CleanupPending@3`, `CleanupPending@4`, or `CleanupPending@5`. Any other pair is `corrupt`, inspect-only, and cannot be migrated or authorize a probe, local cleanup, or remote DELETE. A native v2 record must prove a legal predecessor edge at its recorded state-entry revision; ownership-only revisions may increase the outer revision but cannot change the recorded state-entry history. Missing, contradictory, or skipped history is corrupt. + +An identity-preserving `v1 -> v2` conversion is one legal successor with `revision + 1` and an unchanged remote tuple. The state is unchanged except that every v1 `UploadStarted` maps conservatively to v2 `UploadOutcomeUnknown`. Historical v1 writers could issue PUT while still in `UploadStarted`; no age, fleet version, or current process observation proves that a retained record came from the later pre-PUT-fence writer. It is permitted only when: + +1. every node that can create, commit, recover, heal, or decommission the record advertises both `transition_transaction_v2` and `ilm_recovery_control_v1` for the captured fleet/topology generation, the durable v1-admission stop is active, and the process-epoch drain barrier above is complete; +2. current time is at least the v1 `not_after_unix_nanos` plus five minutes of skew; +3. the canonical path, checksum, full immutable identity, state, remote-version invariant, source record, and ETag all match the observed v1 generation; +4. the migration CAS and strong readback install one fresh recovery lease before any state transition or side effect. + +The original creator must successfully CAS `UploadStarted -> UploadOutcomeUnknown` before issuing remote PUT, and must CAS the exact current owner generation to `LocalCommitStarted` before publishing local metadata. A v2 takeover therefore fences a delayed creator. An implementation that can issue PUT or publish after losing this CAS is not compatible with this protocol. + +After takeover, recovery applies this matrix: + +| State | Approved recovery after exact takeover | Required proof before side effect | +|---|---|---| +| native-v2 `UploadStarted` | CAS `AbortedNoRemote`, then conditionally remove the terminal transaction | Valid native-v2 history proves the creator was fenced before the mandatory pre-PUT `UploadOutcomeUnknown` CAS; migrated v1 never enters this row and no remote request is made | +| `UploadOutcomeUnknown` | Probe under the exact backend lease. Missing becomes terminal cleanup; proven unversioned presence or one nonempty, non-nil exact version becomes `CleanupPending`; nil, ambiguous, or unsupported results become `retained_ambiguous` | Exact transaction/control generations, bounded live probe, current tier destination and lease | +| `Uploaded` | If the exact transitioned tuple or its free-version owns the candidate, remove only the transaction. If the complete original source is still unchanged and no local commit/free-version exists, CAS `CleanupPending`. Otherwise retain | All-pool source/free-version read, full source tuple, bucket incarnation, tier generation, object locks, and post-probe revalidation | +| `LocalCommitStarted` | Exact committed tuple/free-version means record-only cleanup. A fully unchanged original source with no partial committed tuple may move to `CleanupPending`. Missing, divergent, partial, or unavailable metadata is `retained_ambiguous` | Same all-pool proof, including data directory, modification time, size, ETag, transition transaction ID, remote tuple, and destination identity | +| `CleanupPending` | Resume the same exact idempotent candidate delete, or remove only the transaction when local ownership transfer is proven | Current owner lease, source/free-version proof, exact tier lease, physical locks, and before/after fence checks | +| `Committed` | Conditionally remove only the exact terminal transaction when complete local ownership transfer is proven; otherwise classify the record as corrupt or retain it as ambiguous | All-pool strong read matches the complete logical `xl.meta` source identity, transaction ID, remote tuple, destination identity, and any required decommission terminal receipt; no remote DELETE | +| `AbortedNoRemote` | Conditionally remove the exact terminal transaction | Valid native-v2 pre-PUT history or exact migrated v1 `AbortedNoRemote@2`, exact terminal generation, and any required decommission terminal receipt; no remote DELETE | + +Remote DELETE is never admitted directly from `UploadStarted`, `UploadOutcomeUnknown`, `Uploaded`, or `LocalCommitStarted`; it first requires a CAS-protected `CleanupPending` generation with known remote-version semantics. A lost source read, mixed all-pool result, expired lease, failed renewal, or source/control CAS conflict retains the evidence and performs no destructive action. + +New writers emit v2 only after the homogeneous fleet gate, durable v1-admission stop, and process-epoch drain barrier are complete. During a rolling upgrade, new readers accept v1 but all writers continue v1 and no v1 takeover/migration occurs. A v1 reader rejects and retains v2. Downgrade is blocked until v2 creation is disabled and all v2 transactions and recovery-control records are drained or exported; live v2 bytes are never rewritten to v1. + +### Retry and retention policy + +An attempt that loses a CAS or discovers a newer source generation reloads instead of recording a remote failure. A retryable transport timeout, backend 5xx/throttle, metadata quorum outage, or bounded remote-delete failure increments the persisted counters and schedules: + +```text +min(60 seconds * 2^min(consecutive_failure_count - 1, 6), 1 hour) +``` + +A deterministic multiplier from 80 to 100 percent, derived from the source-generation digest and attempt count, is applied to that capped base. The jitter can only shorten the delay and therefore never exceeds the one-hour cap; restarts reproduce the same deadline without synchronizing a fleet. Success or a proven source-state advance resets `consecutive_failure_count` but never decreases `attempt_count`. `next_attempt_at_unix_nanos` is only a not-before scheduler hint; ownership and destructive authority still require the lease and source proofs. + +After 32 consecutive retryable failures or seven days since `first_failure_at_unix_nanos`, whichever occurs first, the control CAS moves to `operator_required` and automatic attempts stop. Unsupported probes and unknown remote-version semantics move directly to `retained_ambiguous`; corrupt source bytes use `corrupt`; incomplete destructive evidence uses `operator_required`. None is periodically hot-looped. An operator may explicitly request another bounded attempt after the underlying capability or configuration changes, but the request creates a new owner lease and preserves the lifetime attempt count. + +This policy bounds automatic work, not evidence lifetime. A source transaction, journal, or manifest is never deleted solely because it is old, numerous, or over a byte threshold. `operator_required` source evidence remains until its protocol reaches a proven terminal state or the legacy-journal disposition below is completed. + +Resolved control tombstones are retained for at least 30 days, immutable export envelopes for at least 90 days, and compact completed disposition receipts for at least 365 days. A collector may conditionally remove only a terminal artifact past its floor after proving that the bound source generation is absent where required, no nonterminal successor or active decommission references it, and the terminal audit checkpoint is durable. Capacity pressure blocks new export/disposition work rather than evicting unexpired or nonterminal evidence. Uncertainty retains the artifact; collection never authorizes source or remote deletion. + +### Legacy journal and manifest disposition + +Journal v1 has neither backend identity nor remote-version authority; v2 has backend identity but still lacks remote-version semantics. Their automatic classification is `retained_ambiguous`, and neither recovery nor an operator action may instantiate a backend or issue remote PUT, GET, probe, or DELETE from those bytes. The approved single-record actions are: + +- **inspect**: strictly decode a server-reconstructed canonical journal identity, perform an all-pool strong read, and return a redacted copy-set/content digest, version, quarantine reason, control classification, age information when known, topology readiness, and decommission coverage; it changes nothing and does not return raw object/version fields by default; +- **export**: after a fresh exact inspect, create-only persist an immutable `rustfs-ilm-recovery-export-v1` envelope containing the raw source bytes and sorted copy manifest, then strongly read it back. The response downloads that envelope rather than rereading the live journal, uses no-store/attachment semantics, and never adds credentials or backend configuration; +- **abandon after export**: v1 or v2 only; create a `Prepared` `rustfs-ilm-recovery-disposition-v1` receipt bound to the immutable export and every source copy, conditionally remove only those exact local journal generations, prove every bound copy absent with no replacement, and advance the receipt through `Applying` to `Completed`. This accepts a possible remote storage leak and never asserts that cleanup occurred. + +Export and abandon require a fresh all-member capability/topology proof including each member's current process epoch; inspect may remain available in a mixed fleet but returns not-ready for mutation. `abandon after export` uses POST and requires `confirm: true`, `action: abandon_remote_cleanup`, `acknowledge_remote_cleanup_abandoned: true`, the export operation ID/digest, source content and copy-set digests, every source ETag, control ETag, and a bounded operator reason code. It is refused while an active decommission or migration receipt covers either record, while physical copy discovery is incomplete, or when the implementation cannot target every discovered copy with its own `If-Match` condition. + +The disposition receipt has immutable action/export/control identities and an immutable sorted copy manifest. Its ETag-CAS generation contains state `Prepared`, `Applying`, or `Completed` and a monotonic sorted `confirmed_absent` set naming only entries from that manifest. Before `Prepared -> Applying`, a fresh all-pool read must find every bound copy at its exact ETag/content digest and repeat the fleet, lock, migration, and decommission checks. The manifest can never be widened, reordered, or replaced. + +During `Applying`, recovery treats each manifest entry independently while retaining the original all-pool boundary. An entry already in `confirmed_absent` must still be strongly absent with no successor or replacement generation. For an unconfirmed entry, an exact ETag/content match may be conditionally deleted and then added to `confirmed_absent` only after strong absence readback. If a crash or lost response left that exact path absent before the progress CAS, recovery may add it only after the same strong absence, stable source/control generation, topology, process-epoch, migration, and decommission proofs establish that no replacement exists. A different ETag/content, an unbound copy, an unreadable member, or loss of any proof is a conflict and preserves the current progress. Thus a crash after deleting copy A but before recording its progress can converge and continue with copy B without requiring deleted copy A to reappear. + +Immediately before each local metadata deletion, the server repeats the applicable all-pool/fleet/lock checks and conditionally targets only the still-unconfirmed exact ETag. Completion requires every immutable manifest entry in `confirmed_absent`, a fresh all-pool proof that all remain absent without replacement, unchanged fleet/process epochs, and no active decommission or migration coverage. A lost final response is success only when strong readback proves the canonical receipt `Completed`. Recovery may repeat only this canonical operation and never creates a tier client or issues a backend request. + +Malformed or unsupported bytes whose outer v1/v2 identity cannot be proven are inspect-only and cannot use abandon. Versions v3-v6 never use `abandon after export`. Their known candidate or manifest ownership must converge through the normal exact protocol. An operator may inspect/export and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or remote-version validation. `Preparing`/`Aborting` manifests may use their existing whole-set rollback; `DispatchAuthorized`/`Completed`, a missing member, a nonempty operation namespace, or any uncertain binding cannot be manually removed. + +### Admin and metrics contract + +The approved surface is single-record and uses a protocol-specific expected tuple; it does not reuse the legacy metadata-reconcile digest or create a bucket/prefix job: + +```text +GET /rustfs/admin/v3/ilm/recovery/records?protocol=&classification=&limit=&marker= +GET /rustfs/admin/v3/ilm/recovery/records/ +POST /rustfs/admin/v3/ilm/recovery/records/ +GET /rustfs/admin/v3/ilm/recovery/exports/ +``` + +List and redacted inspect require `admin:ListTier`. Raw export creation/download, retry, or abandon require `admin:SetTier` because legacy bytes can reveal bucket, object, tier, and remote-version information. The default page is 100 records and the hard maximum is 1,000. A truncated page without a continuation marker is an error, and counts from an incomplete scan are labeled incomplete rather than reported as zero or complete. + +Inspect returns a 15-minute opaque observation receipt bound to the authenticated actor, canonical record identity, source/copy ETags and digests, topology/fleet generation, every member process epoch, action class, issue/expiry time, and nonce. It is observation evidence, not mutation authority. POST requires that receipt and `confirm: true` for terminal actions, and binds the control ETag/revision/classification, requested action, and export digest when applicable. The server repeats all live proofs; a restarted member, membership change, or process-epoch mismatch invalidates the receipt. Client fields are concurrency guards, not authority. Export download reads the immutable envelope, uses TLS plus `Cache-Control: no-store` and attachment disposition, and never logs the raw payload. + +Metrics use bounded labels only: + +- `rustfs_ilm_recovery_records{protocol,classification,schema}` and `rustfs_ilm_recovery_oldest_age_seconds{protocol,classification,schema}`; +- `rustfs_ilm_recovery_attempts_total{protocol,outcome,error_code}`; +- `rustfs_ilm_recovery_operator_actions_total{protocol,action,outcome}`; +- scan completeness, corrupt-record, and orphan-control counters. + +Every label value is a closed enum. `schema` exposes recognized schema identifiers only; an unrecognized raw value maps to `unknown`, while a recognized future schema disabled by the current fleet maps to `unsupported`. `protocol`, `classification`, `outcome`, `error_code`, and `action` likewise map unknown input to one bounded fallback and never expose decoded or operator-provided text. + +Object names, bucket names, tier names, transaction IDs, control IDs, endpoints, ETags, error text, and credentials are never metric labels. Admin output may identify the selected record but redacts credentials and raw backend configuration. Audit/log events use the repository ILM event fields, stable reason codes, authenticated actor, source/control generations, action, and outcome; they do not persist or log provider response bodies. + +One canonical source generation counts once regardless of its physical copy count or how many recovery passes observed it. Attempt counters increment once per coordinator attempt, not per replica, CAS retry, or page revisit. Aggregate totals and oldest age are authoritative only after a complete all-pool scan; partial coverage reports `incomplete` and never publishes a false zero. A legacy record's first-seen time comes from its durable control record rather than an inferred object modification time. + +### Required protocol fixtures + +Implementation acceptance requires deterministic crash/restart and mixed-version fixtures, not timing-only tests. At minimum they cover: + +- a historical v1 `UploadStarted@1` whose PUT may have reached the provider, proving migration yields `UploadOutcomeUnknown` and never `AbortedNoRemote` or a direct delete; +- every accepted v1 state/revision pair above plus checksum-valid impossible pairs such as `CleanupPending@1`, proving impossible history is inspect-only and makes zero backend calls; +- an in-flight v1 creator interleaved with admission stop, process-epoch drain, migration, takeover, and local publication, proving no stale creator can publish after takeover; +- `Committed` with complete, missing, partial, divergent, and unavailable all-pool `xl.meta` ownership proof, proving only the complete exact tuple permits record cleanup; +- an old reader retaining v2, a mixed fleet blocking v2 writer and migration, and downgrade refusing until v2/control records are drained or exported; +- operator abandon across a crash after one per-copy delete, a lost delete response, a lost progress CAS, a replacement ETag, incomplete topology, and active decommission, proving progress is monotonic, replacements survive, unsafe cases retain evidence, and every case issues zero backend PUT, GET, probe, or DELETE calls; +- canonical export replay, a crash before candidate installation, a lost create response, concurrent admission at the remaining-byte boundary, and actor/cluster count, byte, rate, and concurrency exhaustion, proving duplicate IDs consume no new quota, projected totals never oversubscribe, and admission failure mutates no source evidence. ## `xl.meta` free-version boundary @@ -327,9 +472,89 @@ Recursive prefix/delete-all is the exception because physical directory removal - Every code path that removes or overwrites transitioned metadata either atomically leaves an exact free-version owner or enters an already-authorized v6 dispatch. - If both a journal and free-version are observable, recovery preserves the remote object until version-specific authority and all-pool absence prove a single owner. -- Missing `transition-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID. +- Missing `transitioned-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID. -How historical objects without RustFS transition-version-state can be upgraded safely is an **open design**. It requires a read/repair rule with rollback behavior before destructive cleanup may consume those records. +The following single-record protocol approves how historical objects without RustFS `transitioned-version-state` can be upgraded. It does not approve a bulk scanner or allow destructive cleanup to consume an unproven record. + +## Legacy transitioned-version-state reconciliation + +### Current + +An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. The current GET and free-version cleanup paths reject that state rather than interpreting an empty remote version as unversioned. There is no admin route that repairs this field in `xl.meta`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation. + +An explicitly persisted `unknown`, a malformed state, conflicting RustFS/MinIO compatibility keys, an invalid or nil version identifier, and a partial transition tuple are not legacy absence. They remain invalid or ambiguous and fail closed. + +### Approved single-record control surface + +The approved target is one synchronous, exact logical-version operation. It does not list a bucket or prefix and does not create a durable job: + +```text +GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +``` + +`versionId` is required; the literal `null` is the explicit selector for a locally unversioned object, while an omitted or empty selector is invalid. GET requires `admin:ListTier`. It performs an authoritative all-pool metadata read and a bounded server-side live backend probe, but it never writes metadata or mutates the remote tier. Its response includes the canonical immutable source tuple, every original per-set missing-state representation, the proposed target tuple when one is provable, an opaque reconciliation digest over all three, fleet/topology and tier-generation readiness, and whether POST is ready to attempt migration. GET never labels an unmodified legacy record `migrated`. + +POST requires `admin:SetTier`. Its body contains `confirm: true`, the complete source and target tuples, every original per-set representation, and their reconciliation digest returned by a recent GET. The server does not trust a client-supplied state or external assertion: it rereads the object, requires every immutable source field to match exactly after canonicalization, repeats the bounded live probe, and derives the target state itself. For the mutable state/version/destination fields, each set must equal either its original missing-state representation bound by the digest or the exact newly proven target; this is the only accepted partial-retry shape. A missing confirmation, any other stale tuple or digest, or a widened selector is rejected before any write. + +The expected tuple binds all evidence whose change could redirect the repair: + +- bucket name and incarnation; exact object name, local version ID, data directory, modification time, size, and ETag; +- transition completion status, tier name, canonical remote object name, raw remote-version key presence/value, and raw state-key presence/value under both compatibility prefixes; +- tier-config generation and the `transition-tier-destination-id` binding, including backend type, endpoint/bucket/prefix identity, and the credential-independent backend fingerprint; +- topology generation and the generation/digest of every authoritative `xl.meta` copy found across pools and sets. + +The only allowed state derivations are: + +| Live proof | Persisted state | Persisted remote version | Remote request meaning | +|---|---|---|---| +| Provider proves versioning disabled and the candidate is present | `KnownDisabled` | Absent/empty | Send no `versionId` | +| Provider proves suspended-version null semantics and the exact candidate is present | `SuspendedNull` | Literal `null` | Send the provider's null-version form | +| Provider proves one exact, nonempty, non-`null` opaque version | `Exact` | That exact opaque identifier | Send that exact `versionId` | + +Missing, multiple, changing, or unsupported probe results do not select a state. In particular, a preexisting empty remote-version field is not evidence for `KnownDisabled`, and a client may not nominate `Exact` or `SuspendedNull`. + +The POST may write only the derived `transitioned-version-state`, its corresponding `transitioned-versionID` value when the proven model requires one, and the exact `transition-tier-destination-id` binding under both compatibility prefixes in the matching `xl.meta` version. It does not issue remote GET beyond the proof probe, remote PUT, remote DELETE, local object DELETE, free-version cleanup, transaction/journal cleanup, tier-config mutation, restore, or source-payload rewrite. Reconciliation establishes metadata meaning; a later ordinary owner may perform cleanup under its own destructive protocol. + +### Outcome contract + +POST returns exactly one of the following outcomes and whether it changed bytes. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged: + +| Outcome | Meaning and permitted effect | +|---|---| +| `migrated` | All authoritative copies already contain, or were monotonically advanced to, the same proven state and destination identity. Only this outcome makes the record eligible for later ordinary read/delete semantics. | +| `retained-ambiguous` | The tuple is structurally legacy-compatible, but the live probe is missing, multiple, changing, unsupported, or otherwise cannot prove exactly one state. No metadata or remote object is changed. | +| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established, and nothing is changed. | +| `backend-unavailable` | The bound tier generation/destination cannot be acquired, the bounded probe fails, or a metadata quorum/strong readback needed to complete the operation is unavailable. Any already-persisted monotonic subset is retained for an idempotent retry; it is never rolled back. | + +HTTP failure detail may distinguish a stale expected tuple, lost fence, timeout, or unavailable quorum, but it must preserve one of these machine-readable outcomes. Logs and audit events include request identity, object identity, tier, generations, outcome, and whether bytes changed; they never include credentials or raw credential-derived configuration. + +### Fence, write, and retry order + +The approved POST executes the following order. A step that cannot be proven stops the operation without remote mutation: + +1. Authenticate `admin:SetTier`, validate the exact single-record selector, `confirm: true`, expected tuple, and digest. +2. Prove every required node advertises the reconciliation format and destination-identity capability; capture the fleet and topology generation. Unknown or unsupported nodes block the writer. +3. Acquire the bucket-lifecycle WRITE fence and validate the bucket incarnation. +4. Acquire the exact tier-config generation lease bound to the expected destination identity. +5. Acquire exact object-version WRITE locks for every owning physical pool/set in stable pool/set order. +6. Perform an authoritative all-pool read, reject duplicate/conflicting ownership, validate every compatibility key, and require each set to match the immutable source tuple plus either its digest-bound original missing-state representation or the exact proposed target. +7. Run one bounded, cancellation-aware live probe through the leased backend. No client or cached probe result is authority. +8. Before writing, revalidate the fleet/topology generation, bucket incarnation and lifecycle fence, tier lease/destination identity, physical owner set, and complete metadata tuple. +9. Write the same derived state and destination identity to each authoritative set with that set's metadata quorum and conditional generation. A timeout or response loss is resolved only by a strong read of that exact set. +10. Strongly reread every authoritative set and revalidate the full tuple, state, destination identity, and topology before returning `migrated`. Release locks and leases in reverse order. + +Cross-pool and cross-set partial success is monotonic. The only legal repair edge is `missing state -> one proven {state, remote version, destination identity}`. A retry may accept an already-written subset only when every known copy equals the newly proven target, every remaining copy equals its original missing-state representation captured by the reconciliation digest, and all immutable source fields still match; it then fills only the missing copies. This exact target-plus-original subset is neither stale nor corrupt. The retry never clears a known state, rewrites it to another state, changes destination identity, or rolls a successful set back to missing/`Unknown`. Any other divergent value produces `corrupt`; an unavailable set/readback produces `backend-unavailable`, and destructive cleanup remains blocked until a later strong all-pool read proves complete convergence. + +GET takes the same fleet/topology snapshot and authoritative all-pool read but no write locks that imply mutation authority. Because GET is advisory, POST always repeats every fence, read, and live proof rather than promoting the GET result. + +### Mixed-version and future batch work + +The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. Downgrade is blocked while reconciled records could be rewritten by readers that erase or misinterpret those fields. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient. + +Explicit `Unknown`, corruption, and ambiguity remain fail closed for reads that cannot prove non-destructive semantics and for every destructive path. A migrated record becomes ordinary explicit metadata, but reconciliation itself never transfers remote DELETE ownership. + +A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a separate durable job identity, create-only admission, lease/CAS checkpoint, bounded pages, per-record expected tuples and outcomes, cancellation/restart semantics, retention, fleet rollout negotiation, and status counters. Implementations must not approximate that protocol by adding a list selector or background loop to the synchronous route. ## Durable namespace receipts during decommission @@ -337,7 +562,7 @@ How historical objects without RustFS transition-version-state can be upgraded s Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads. -The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. +The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current exact ETag conditional-delete primitive. Completion verifies every expected receipt and target checkpoint before the source pool can be removed. A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target. @@ -347,7 +572,7 @@ Receipts have no enum state. Their legal evolution is `absent -> checkpoint -> m | Observed state | Unique current owner | Current recovery decision | Destructive admission | |---|---|---|---| -| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. An eligible v6 journal/manifest record is directly removed by exact ETag; transition-transaction cleanup follows its documented current unconditional path | Receipt state grants no remote-delete authority | +| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. Eligible v6 journal/manifest and transition-transaction records are removed only by their exact observed ETag | Receipt state grants no remote-delete authority | | Source and target exact identity/checkpoint agree | Decommission coordinator for the run token | Create or CAS-advance the run-scoped receipt | Successor must be monotonic and topology-bound where required | | Receipt already covers the same successor | Decommission coordinator for the run token | Treat as idempotent | Exact identity/checkpoint only | | Conflicting receipt, checksum/schema/path error, divergent target record, missing ETag, or non-successor checkpoint | No decommission actor acquires cleanup authority | Fail decommission and retain source | Never overwrite or guess | @@ -371,7 +596,7 @@ Receipt and expected-manifest create/CAS conflicts retry at most three times. Ex ### Approved target failure matrix -The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. In particular, transition-transaction initial and successor writes and deletes are currently unconditional, while the transition transaction's initial write and the manual job's initial write have neither create-only installation nor mandatory lost-response strong-readback convergence. +The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. Transition transaction v1 now has create-only installation, exact ETag successor CAS, and conditional terminal deletion, but it still lacks mandatory lost-response exact-successor readback and a renewable durable recovery lease. The manual job's initial write remains non-create-only. | Event | Approved result | |---|---| @@ -382,19 +607,20 @@ The matrix below is the normative approved target, not a blanket description of | Crash after remote DELETE but before journal/free-version cleanup | Retry the same exact idempotent DELETE under the same fences, then conditionally clean local evidence | | Cancellation | Stop issuing new work, persist monotonic cancellation where the protocol has it, and leave ambiguous durable records for recovery. Cancellation is never rollback proof after authorization | | Rolling upgrade | Gate writers on the minimum capability required by the format. Known older journal/RPC versions follow their explicit compatibility rule; unknown formats are retained | -| Downgrade | Drain v6 journals before removing all v6-aware workers. Do not write a new format until its downgrade reader behavior and writer gate are specified | +| Downgrade | Drain v6 journals and any enabled transition-v2/control protocol before removing their capable workers. Do not write a new format until its downgrade reader behavior and writer gate are specified | | Corrupt or unknown input | Record a diagnosable failure, retain bytes, and block destructive action/completion | -Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have a complete persisted-format negotiation for rolling downgrade. Until one is designed, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent. +Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have an implemented persisted-format negotiation for rolling downgrade. The approved transition-v2/control gate above is not current behavior. Until the applicable gate is implemented, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent. ### Current format compatibility decisions | Family/version | Current reader and writer behavior | Upgrade, downgrade, and ignore rule | |---|---|---| -| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. A future schema needs a fleet writer gate and an old-reader retention test before rollout; downgrade behavior is open | +| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. V1 remains the only writer format until the approved v2 fleet gate is implemented; a v2 reader never rewrites an active v1 record | +| Transition transaction v2 and recovery-control/export/disposition v1 | Approved target only; no current reader or writer emits these formats | Roll out read support before the homogeneous writer gate; old readers reject and retain. Disable creation and prove all active records drained before downgrade; never rewrite v2 to v1 | | Tier mutation intent v1; peer RPC v3/v4 | Durable readers/writers require intent v1. New peers accept signed/canonical v3 and v4 RPC; old v3 peers return an exact authenticated unsupported response to v4 | Pause and drain edit/remove/clear across the mixed interval; do not automatically retry v4 as v3. Unknown durable intent is retained and blocks recovery | | Manual job/scope/task/result v1 | Writers emit the v1 family. Manual-job runtime recovery accepts an uppercase UUID path when both shard strings match its uppercase prefix, then loads the lowercase canonical job by UUID; the decommission validator recomputes the canonical path and rejects that alias. Other decoder/path/checksum failures stop reconciliation. Runtime capabilities advertise `enqueue_only` and `async`, but the async run handler does not consult a fleet capability gate and a direct request creates a job | Runtime recovery still needs exact lowercase canonical-path validation to prevent alias-driven duplicate work. Caller/operator orchestration must verify every required node and fail closed when capability is unknown or unsupported. An automatic server-side fleet gate and persisted downgrade negotiation remain open; unknown records are never ignored as completed work | -| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled | +| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Never translate empty version ID to known-disabled or authorize remote DELETE. Retain unless the approved exact inspect/export/abandon protocol conditionally removes only the local journal generation | | Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof | | Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization | | Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation | @@ -404,10 +630,10 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr | Protocol | Current operator/telemetry surface | Current retention | Required follow-up | |---|---|---|---| -| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Backlog age/count/state metrics, bounded policy, and durable takeover status | +| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Implement the approved v2 lease/takeover, recovery-control status, bounded retry, and backlog metrics | | Tier mutation intent | Admin mutation response plus recovery diagnostics; no dedicated reconcile API | Peer aborted tombstone through expiry plus skew; ambiguous coordinator/peer records retained | Status/reconcile view for mutation, peer convergence, config generation, and blocked tiers | | Manual job | POST run response, GET status, DELETE cancel; runtime capabilities advertise both modes | Job/task/result history is indefinite. Terminalizers only best-effort delete the exact scope; startup skips a terminal job with a leftover scope, which remains until a later admission claimant lazily replaces it | Age/count/bytes limit and a terminal-history/scope GC protocol that preserves recovery evidence | -| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Safe operator inspection/disposition, backlog age/count by version/state, bounded recovery without evidence loss | +| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Implement the approved single-record inspect/export/disposition, bounded retry controls, and logical backlog metrics | | Decommission receipt | Decommission state/events including `receipt_cleanup_failed` | Completion triggers only best-effort receipt/manifest cleanup. Delete failures reported as `receipt_cleanup_failed`, as well as abandoned runs, can leave run-scoped records behind | Run-scoped retention and resume-safe cleanup policy | Retention is a protocol transition, not raw deletion. Any collector must name its unique owner, minimum age/count/bytes bound, exact terminal or quarantine predicate, readback behavior, decommission interaction, and audit/metric output. It may not collect a record solely because it is old. diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 44b9de648..813b3aefb 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -39,11 +39,11 @@ Internal metadata is stored under both `x-rustfs-internal-` and `x-minio |--------|---------| | `transition-status` | `"complete"` when tiered | | `transitioned-object` | tier key path (stored without the tier prefix; `get_dest` adds it) | -| `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) | +| `transitioned-versionID` | Provider version identifier: current exact UTF-8 text, legacy RustFS raw UUID bytes, MinIO's empty unversioned value, or absent for some historical unversioned records. Interpret it only with `transitioned-version-state` or a live compatibility probe. | | `transition-tier` | tier name | | `tier-free-versionID` | delete-marker version for free-version sweep | -Reading binary values must reject empty, malformed, and nil values (regression covered in `crates/filemeta/src/filemeta/version.rs` tests): +Legacy raw UUID values must reject empty, malformed, and nil UUIDs (regression covered in `crates/filemeta/src/filemeta/version.rs` tests): ```rust get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) @@ -52,7 +52,7 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) // None for: absent key, wrong-length bytes, nil UUID ``` -`transition_version_id == None` means the tier bucket is unversioned; the GET/DELETE against the tier must then send no `versionId` parameter. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. +`transition_version_id == None` means only that no usable legacy UUID projection exists; it does not prove the remote bucket's versioning model. Only an explicit `KnownDisabled` state authorizes ordinary GET/DELETE to omit `versionId`. A missing state with an absent or empty version key remains `Unknown` and requires the bounded compatibility probe or the approved reconcile workflow; it never directly authorizes cleanup. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. ## Inspect xl.meta directly @@ -64,8 +64,8 @@ cargo build -p rustfs-filemeta --example dump_fileinfo | Output | Meaning | |---|---| -| `transition_ver_id: ` | No versionId will be sent to the tier (correct for a non-versioned tier bucket). | -| `transition_ver_id: ` | That UUID will be sent as `?versionId=`. | +| `transition_ver_id: ` | No usable legacy UUID projection exists. Inspect `transitioned-version-state` and the raw compatibility keys; do not infer unversioned semantics. | +| `transition_ver_id: ` | A legacy UUID representation decoded successfully. It is not destructive authority unless the persisted state or reconcile proof establishes the exact remote model. | There is one `xl.meta` per erasure shard disk (`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should be identical. `dump_versions` (same crate) lists every version in a file. @@ -80,7 +80,7 @@ RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs ... | `fetching transitioned object from tier` | DEBUG | Emitted before the tier request. | | `tier GET failed` | ERROR | Includes `tier_version_id`. | -If both `x-rustfs-internal-transitioned-versionID` and `x-minio-internal-transitioned-versionID` are the empty string, the object was transitioned to a non-versioned tier bucket and no versionId must be sent. +If the version keys are empty while `transitioned-version-state` is absent, the record has the historical MinIO unversioned shape but still remains `Unknown`; only the compatibility probe or reconcile protocol may prove `KnownDisabled`. If state is explicitly `KnownDisabled`, no `versionId` is sent. ## Manual transition run @@ -154,6 +154,89 @@ Historical transition transactions in `upload_outcome_unknown` state can use an `finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically. +## Inspect and disposition retained recovery records + +This section describes an **approved target that is not implemented yet**. Current servers do not expose the routes below and continue to quarantine tier-delete journal v1/v2 records. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees. + +The approved read-only inventory is bounded and paginated: + +```text +GET /rustfs/admin/v3/ilm/recovery/records?protocol=&classification=&limit=&marker= +GET /rustfs/admin/v3/ilm/recovery/records/ +``` + +List and redacted inspect require `admin:ListTier`. The server reconstructs the canonical source identity, strongly reads every authoritative copy, and reports one logical record with its schema, classification (`retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`), stable reason code, copy/content digests, retry deadline/counters, fleet readiness, scan completeness, and decommission coverage. It does not return raw legacy bytes, object/version names, endpoints, credentials, or provider error text in the default JSON. Incomplete pool coverage, divergent copies, a missing ETag, corruption, or a truncated page without a continuation marker is fail-closed and cannot produce an actionable receipt. + +Inspect returns a 15-minute opaque observation receipt. It binds the authenticated actor, canonical record, every source copy/ETag/digest, topology/fleet generation, requested action class, issue/expiry time, and nonce. The receipt prevents a stale request from widening its target; it is not cleanup authority. + +For a strictly decoded v1/v2 tier-delete journal, the approved evidence-preserving flow is: + +1. Inspect the exact record and independently decide whether retaining the local cleanup obligation is still useful. +2. With `admin:SetTier`, create an immutable server-side export from the current observation receipt. The export contains the exact raw journal bytes and copy manifest, is installed create-only at the canonical digest-derived export ID, strongly read back, and downloaded through a no-store attachment response: + + ```text + POST /rustfs/admin/v3/ilm/recovery/records/ + { "action": "export", "observation_receipt": "" } + + GET /rustfs/admin/v3/ilm/recovery/exports/ + ``` + +3. Only after preserving that export, submit a fresh exact disposition with `admin:SetTier`: + + ```json + POST /rustfs/admin/v3/ilm/recovery/records/ + { + "action": "abandon_remote_cleanup", + "confirm": true, + "acknowledge_remote_cleanup_abandoned": true, + "observation_receipt": "", + "export_id": "", + "export_sha256": "", + "reason_code": "" + } + ``` + +The last action removes only the exact local v1/v2 journal generations by per-copy `If-Match` after a durable `Prepared` disposition receipt and fresh all-member capability proof. The receipt advances `Prepared -> Applying -> Completed` and records a monotonic per-copy `confirmed_absent` set. If the server deletes copy A and crashes before recording progress, recovery may confirm A absent under the unchanged source/control, topology, process-epoch, migration, and decommission proofs, persist that progress, and continue with still-exact copy B. A replacement ETag is always a conflict; recovery never widens the immutable copy manifest. + +The action never creates a tier client, probes a backend, or issues remote PUT/GET/DELETE. Its meaning is deliberately narrow: the operator accepts that remote storage may leak and abandons RustFS cleanup after preserving evidence. A changed copy, active decommission, missing member, topology/process restart, incomplete read, or uncertain replacement proof retains the evidence. Success requires every bound copy be durably confirmed absent, a fresh all-member/decommission proof, and the disposition receipt durably `Completed`; response loss resumes only the same canonical operation ID. + +Canonical replay of an identical export/disposition consumes no new quota. New operations require a complete artifact inventory and are refused before source mutation when the projected retained total, including the fully encoded candidate, would exceed 10,000 exports, 10,000 disposition receipts, 1 GiB of encoded export data, or 256 MiB of encoded control/disposition data. The quota decision, create-only installation, and exact readback share one cluster-scoped admission WRITE lock. That lock is always acquired before control/source/disposition and physical metadata locks and is released before disposition `Applying` or any source deletion; callers never acquire it while holding those inner guards. A crash before installation consumes no capacity, and lost installation response is resolved by canonical readback under the same serialized order, so concurrent nodes cannot oversubscribe a stale snapshot. Admission is also limited to ten new creations per actor per minute, 100 cluster-wide per minute, 32 concurrent exports, and eight concurrent dispositions. Capacity pressure never evicts recovery evidence or blocks ordinary object I/O; the collector examines at most 100 terminal artifacts per minute. + +Malformed/unsupported records and journal v3-v6 cannot use abandon. Known-version and v6 manifest ownership must converge through their normal exact recovery protocol. Operators may inspect, export, and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or version semantics. + +Automatic retry state survives restart. Retryable transport/quorum failures use a 60-second exponential base capped at one hour and a deterministic 80-to-100-percent multiplier, so jitter never increases the capped delay. After 32 consecutive failures or seven days from the first persisted failure, automatic work stops at `operator_required`. Unsupported or ambiguous evidence goes directly to `retained_ambiguous`/`operator_required`; age alone never deletes it. Resolved controls, immutable exports, and completed disposition receipts have minimum 30-day, 90-day, and 365-day retention respectively, and are collected only after exact source absence, decommission, successor, and audit checks. + +The full schema, lease, mixed-version, retry, privacy, and metric requirements are in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#bounded-recovery-control-and-operator-disposition). + +## Reconcile legacy transition-version metadata + +This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version. + +The approved interface is synchronous and accepts exactly one bucket/object/local-version tuple: + +```text +GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= +``` + +`versionId` is required; use the literal `null` for a locally unversioned object. An omitted or empty selector is invalid. GET requires `admin:ListTier`. It reports the authoritative all-pool tuple, destination identity, fleet/topology readiness, live probe classification, opaque expected-tuple digest, and a machine-readable diagnosis. It returns `ready-to-migrate`, not `migrated`, when a missing state is provable because GET is read-only. + +POST requires `admin:SetTier`, `confirm: true`, and the complete immutable source tuple, original per-set missing-state representations, proposed target, and reconciliation digest returned by GET. The server rereads every authoritative copy and repeats the bounded live backend probe; provider console output or an operator-supplied state is diagnostic evidence only, never write authority. A retry accepts only copies that still match their digest-bound original representation or already equal the exact proven target; any other divergence is stale or corrupt. The server may persist only one of these exact state/version pairs, together with the bound destination identity: + +| Proven remote model | State | Version value | +|---|---|---| +| Versioning disabled | `KnownDisabled` | Empty/absent; later requests omit `versionId` | +| Versioning suspended null object | `SuspendedNull` | Literal `null` | +| One exact version | `Exact` | Exact nonempty, non-`null` opaque identifier | + +The response outcome is `migrated`, `retained-ambiguous`, `corrupt`, or `backend-unavailable`. `migrated` means strong all-pool readback proved the same state and destination identity on every authoritative copy; it can be idempotent with `changed=false`. Ambiguous/missing/multiple probe results are retained, and explicit `Unknown`, malformed or conflicting dual keys, nil identifiers, partial tuples, or copies outside the exact `{original missing representation, proven target}` retry subset fail closed. An unavailable backend, tier generation, metadata quorum, or required strong readback reports `backend-unavailable`; a monotonic partial write is retained for retry and never rolled back. + +The POST does not issue remote DELETE or PUT, remove local data, create a free-version, clean a transaction/journal, or change tier configuration. It holds the approved fleet/topology, bucket-lifecycle, exact tier-generation/destination, and stable all-pool object-version fences across authoritative reread and the bounded probe; it rechecks them before quorum writes and after strong readback. A fleet containing a node that cannot preserve the explicit state/destination binding is inspect-only, and a cross-pool first match is never enough. + +There is intentionally no bucket, prefix, or fleet selector. Batch repair requires a separate durable, resumable job protocol and remains future work. Until the single-record route is implemented, retain affected metadata, use external inspection only for diagnosis, and never hand-edit `xl.meta` or enable remote cleanup by assuming that an empty version field means an unversioned tier. + +The full approved fence, quorum, cross-set retry, destination-binding, and mixed-version contract is specified in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#legacy-transitioned-version-state-reconciliation). + ## Invariant: local-first expiry ordering `expire_transitioned_object` deletes local metadata first (making the object unreachable) and leaves a persisted free-version for remote-tier cleanup. `tier_free_version_recovery.rs` scans and re-enqueues that record; the lifecycle worker's `cleanup_free_version_exact` in `bucket_lifecycle_ops.rs` performs the fenced remote delete, local-marker cleanup, and rescan. Never remove a remote tier version while live local metadata still points at it: doing so lets a concurrent GET read a stored version_id whose remote version is already gone and fail with `NoSuchVersion`. diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 535554a46..310212cc4 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -80,6 +80,82 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result { .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration")) } +fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> { + match config.tier_type { + TierType::S3 => { + let _ = config + .s3 + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?; + } + TierType::Wasabi => config.name = wasabi_payload_name(config)?, + TierType::RustFS => { + config.name = config + .rustfs + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))? + .name + .clone(); + } + TierType::MinIO => { + config.name = config + .minio + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))? + .name + .clone(); + } + TierType::Aliyun => { + config.name = config + .aliyun + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))? + .name + .clone(); + } + TierType::Tencent => { + config.name = config + .tencent + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))? + .name + .clone(); + } + TierType::Huaweicloud => { + config.name = config + .huaweicloud + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))? + .name + .clone(); + } + TierType::Azure => { + config.name = config + .azure + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))? + .name + .clone(); + } + TierType::GCS => { + let _ = config + .gcs + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?; + } + TierType::R2 => { + config.name = config + .r2 + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))? + .name + .clone(); + } + TierType::Unsupported => {} + } + Ok(()) +} + fn spawn_transition_tier_config_propagation(action: &'static str) { if let Some(notification_sys) = current_notification_system() { debug!( @@ -263,75 +339,7 @@ impl Operation for AddTier { let mut args: TierConfig = serde_json::from_slice(&body) .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?; - match args.tier_type { - TierType::S3 => { - args.name = args - .s3 - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))? - .name; - } - TierType::Wasabi => { - args.name = wasabi_payload_name(&args)?; - } - TierType::RustFS => { - args.name = args - .rustfs - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))? - .name; - } - TierType::MinIO => { - args.name = args - .minio - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))? - .name; - } - TierType::Aliyun => { - args.name = args - .aliyun - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))? - .name; - } - TierType::Tencent => { - args.name = args - .tencent - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))? - .name; - } - TierType::Huaweicloud => { - args.name = args - .huaweicloud - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))? - .name; - } - TierType::Azure => { - args.name = args - .azure - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))? - .name; - } - TierType::GCS => { - args.name = args - .gcs - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))? - .name; - } - TierType::R2 => { - args.name = args - .r2 - .clone() - .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))? - .name; - } - _ => (), - } + normalize_add_tier_payload_name(&mut args)?; debug!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, @@ -1150,6 +1158,44 @@ mod tests { assert_eq!(err.message(), Some("missing Wasabi configuration")); } + #[test] + fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() { + for (provider, wire) in [ + ( + "S3", + serde_json::json!({ + "Type": "s3", + "Name": "COLD-S3", + "S3": { + "Endpoint": "https://s3.example.invalid", + "AccessKey": "access", + "SecretKey": "secret", + "Bucket": "archive" + } + }), + ), + ( + "GCS", + serde_json::json!({ + "Type": "gcs", + "Name": "COLD-GCS", + "GCS": { + "Endpoint": "https://storage.googleapis.com", + "Creds": "e30=", + "Bucket": "archive" + } + }), + ), + ] { + let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode"); + let expected = config.name.clone(); + + normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary"); + + assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared"); + } + } + #[test] fn resolve_tier_name_prefers_path_parameter() { let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse"); @@ -1756,5 +1802,11 @@ mod tests { } assert!(!production.contains("check_key_valid(get_session_token")); + + let add_tier = source_block(production, "impl Operation for AddTier"); + assert!( + add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"), + "AddTier must preserve canonical top-level provider names through the tested boundary helper" + ); } } diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index 5360948e3..d5751d5c4 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -170,12 +170,12 @@ use s3s::dto::{ DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts, GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput, - PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm, - SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption, - ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, - WebsiteRedirectLocation, + PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus, + SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, + ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, + Timestamp, TimestampFormat, WebsiteRedirectLocation, }; -use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; +use s3s::header::X_AMZ_RESTORE; use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; diff --git a/rustfs/src/app/object/restore.rs b/rustfs/src/app/object/restore.rs index 56319265d..96201a670 100644 --- a/rustfs/src/app/object/restore.rs +++ b/rustfs/src/app/object/restore.rs @@ -165,18 +165,49 @@ impl DefaultObjectUsecase { validate_table_catalog_object_mutation(&bucket, &object).await?; - let rreq = rreq.ok_or_else(|| { - S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required") - })?; + // Typed S3 errors on every RestoreObject failure (backlog#2205): a + // `Custom` code serializes as a generic 500, which makes SDK clients + // retry client errors and conflicts alike. + let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?; + + // SELECT-type restore is not supported (backlog#1341). The restore + // path can only write the retrieved bytes back to the source key, so + // honouring a SELECT request overwrote the source object with + // SELECT-only metadata (dropping `x-amz-restore`, user metadata and + // tags on an unversioned bucket, or publishing a bogus latest version + // on a versioned one) while never writing anything to + // `OutputLocation.S3`. Reject before any guard, metadata write or + // fabricated `x-amz-restore-output-path` response header. + if rreq + .type_ + .as_ref() + .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT) + { + return Err(S3Error::with_message( + S3ErrorCode::NotImplemented, + "SELECT restore requests are not supported.", + )); + } let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + // Validate the request shape before taking any lock or reading the + // object: a malformed request or an illegal `Days` value is a client + // error, and the validator messages are static — they carry no + // backend or credential detail. + if let Err(e) = validate_restore_request(&rreq, store.clone()) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Restore object validation failed: {e}"), + )); + } + let version_id_str = version_id.clone().unwrap_or_default(); let mut opts = post_restore_opts(&version_id_str, &bucket, &object) .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?; + .map_err(ApiError::from)?; apply_bucket_generation_guard(&req, &bucket, &mut opts)?; // `apply_bucket_generation_guard` deliberately tolerates a missing guard // (only the S3 access layer installs one), so this must not hard-require @@ -192,11 +223,7 @@ impl DefaultObjectUsecase { } }; - // SELECT-type restores skip both the ongoing check and the metadata - // write below, so the accept guard would protect nothing for them — - // they keep the plain (read-locked) accept path. - let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT"); - let restore_operation_id = (!is_select).then(Uuid::new_v4); + let restore_operation_id = Some(Uuid::new_v4()); let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id { Some( store @@ -210,10 +237,10 @@ impl DefaultObjectUsecase { // Hold the restore-accept guard across the restore-status read, the // ongoing/already-restored decision, and the metadata write below, so - // two concurrent (non-SELECT) POST ?restore cannot both observe - // ongoing=false and both start a copy-back (backlog#1304). Reads and - // writes inside this scope run with no_lock; the guard is dropped - // before the copy-back is spawned so it never blocks readers. + // two concurrent POST ?restore cannot both observe ongoing=false and + // both start a copy-back (backlog#1304). Reads and writes inside this + // scope run with no_lock; the guard is dropped before the copy-back is + // spawned so it never blocks readers. // Contention on the accept guard (e.g. a concurrent accept or an // in-flight commit on the same object) is transient — answer 503 // SlowDown so SDK clients back off and retry instead of treating it @@ -222,9 +249,7 @@ impl DefaultObjectUsecase { if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id { return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into()); } - let mut accept_guard = if is_select { - None - } else { + let mut accept_guard = { let guard = store .acquire_restore_accept_guard(&bucket, &object) .await @@ -233,24 +258,17 @@ impl DefaultObjectUsecase { Some(guard) }; - let mut obj_info = store - .get_object_info(&bucket, &object, &opts) - .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?; + // A missing key or version must stay NoSuchKey / NoSuchVersion, and an + // authorization or storage failure must keep its own identity, so map + // the storage error instead of flattening it (backlog#2205). + let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; - // Check if object is in a transitioned state + // Restoring an object that was never transitioned is the S3 + // InvalidObjectState case, not an internal error. if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrInvalidTransitionedState".into()), - "restore object failed.", - )); - } - - // Validate restore request - if let Err(e) = validate_restore_request(&rreq, store.clone()) { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrValidRestoreObject".into()), - format!("Restore object validation failed: {}", e), + S3ErrorCode::InvalidObjectState, + "The operation is not valid for the object's storage class.", )); } @@ -260,7 +278,7 @@ impl DefaultObjectUsecase { // would create an ABBA cycle. If the probe succeeds, reacquire and // re-read the object before replacing the exact orphan generation. let mut superseded_worker_guard = None; - if obj_info.restore_ongoing && !is_select { + if obj_info.restore_ongoing { match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) { OngoingRestoreRecovery::ActiveOrUnsafe => { return Err(S3Error::with_message( @@ -293,13 +311,11 @@ impl DefaultObjectUsecase { .map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?, ); opts.no_lock = true; - obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| { - S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.") - })?; + obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrInvalidTransitionedState".into()), - "restore object failed.", + S3ErrorCode::InvalidObjectState, + "The operation is not valid for the object's storage class.", )); } if obj_info.restore_ongoing { @@ -327,11 +343,11 @@ impl DefaultObjectUsecase { remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID); remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK); - let mut header = HeaderMap::new(); - let event_object_info = obj_info.clone(); let obj_info_ = obj_info.clone(); - if !is_select { + // Scopes the accept-guarded metadata write: everything below runs + // inside the accept critical section, which is released right after. + { obj_info.metadata_only = true; metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string()); let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| { @@ -403,7 +419,7 @@ impl DefaultObjectUsecase { &restore_dst_opts, ) .await - .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?; + .map_err(ApiError::from)?; rustfs_scanner::record_dirty_usage_bucket(&bucket); #[cfg(test)] maybe_pause_after_restore_status_commit(&bucket, &object).await; @@ -429,17 +445,6 @@ impl DefaultObjectUsecase { drop(accept_guard); drop(restore_bucket_lifecycle_guard); - // Handle output location for SELECT requests - if let Some(output_location) = &rreq.output_location - && let Some(s3) = &output_location.s3 - && !s3.bucket_name.is_empty() - { - let restore_object = Uuid::new_v4().to_string(); - if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() { - header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value); - } - } - // Spawn restoration task in the background. Pin the copy-back to the // version the accept resolved and flagged: with a versionless request // on a versioned bucket, a PUT landing between the accept and the @@ -499,7 +504,7 @@ impl DefaultObjectUsecase { restore_output_path: None, }; helper = helper.object(event_object_info).version_id(version_id_str); - let result = Ok(S3Response::with_headers(output, header)); + let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); result } @@ -629,6 +634,28 @@ mod tests { assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe); } + fn restore_request(days: Option) -> RestoreRequest { + RestoreRequest { + days, + description: None, + glacier_job_parameters: None, + output_location: None, + select_parameters: None, + tier: None, + type_: None, + } + } + + fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput { + RestoreObjectInput::builder() + .bucket(bucket.to_string()) + .key(key.to_string()) + .restore_request(Some(rreq)) + .build() + .expect("restore input should build") + } + + /// backlog#2205: a missing restore body is a client error, not a 500. #[tokio::test] async fn execute_restore_object_rejects_missing_restore_request() { let input = RestoreObjectInput::builder() @@ -641,10 +668,92 @@ mod tests { let usecase = DefaultObjectUsecase::without_context(); let err = usecase.execute_restore_object(req).await.unwrap_err(); - match err.code() { - S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"), - code => panic!("unexpected error code: {:?}", code), - } + assert_eq!(err.code(), &S3ErrorCode::MalformedXML); + } + + /// backlog#1341: a SELECT restore must be rejected outright — the restore + /// path can only write back to the source key, never to + /// `OutputLocation.S3`. Rejection happens before the store is resolved, so + /// an uninitialized usecase still answers NotImplemented rather than the + /// InternalError every request that gets past this point returns. + #[tokio::test] + async fn execute_restore_object_rejects_select_type() { + let mut rreq = restore_request(None); + rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT)); + + let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST); + let usecase = DefaultObjectUsecase::without_context(); + + let err = usecase.execute_restore_object(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + /// backlog#2205: every RestoreObject failure that reaches storage must + /// keep its typed S3 identity. Before this, a missing key, a malformed + /// version-id, an illegal `Days` and an object that was never transitioned + /// all collapsed into `Custom(...)` codes, which serialize as a retryable + /// HTTP 500. + #[tokio::test] + #[serial_test::serial] + async fn execute_restore_object_maps_failures_to_typed_s3_errors() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let context = crate::app::gating_test_env::shared_gating_ambient().await; + let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create restore test bucket"); + let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec()); + store + .put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default()) + .await + .expect("put untransitioned test object"); + + let usecase = DefaultObjectUsecase::with_context(Some(context)); + + // An illegal `Days` is a client error, rejected before any lock or + // object read. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "local-object", restore_request(Some(0))), + Method::POST, + )) + .await + .expect_err("days=0 must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + // A malformed version-id keeps InvalidArgument instead of being + // flattened inside `post_restore_opts`. + let mut input = restore_input(&bucket, "local-object", restore_request(Some(1))); + input.version_id = Some("not-a-uuid".to_string()); + let err = usecase + .execute_restore_object(build_request(input, Method::POST)) + .await + .expect_err("malformed version-id must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + + // A missing key stays NoSuchKey. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "missing-object", restore_request(Some(1))), + Method::POST, + )) + .await + .expect_err("missing key must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchKey); + + // Restoring an object that was never transitioned is the S3 + // InvalidObjectState case, not an internal error. + let err = usecase + .execute_restore_object(build_request( + restore_input(&bucket, "local-object", restore_request(Some(1))), + Method::POST, + )) + .await + .expect_err("untransitioned object must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState); } #[tokio::test] diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index e90f9d9ae..19548aa66 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -368,7 +368,8 @@ pub(crate) mod bucket { version_id: &str, bucket: &str, object: &str, - ) -> Result { + ) -> Result + { crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts( version_id, bucket, object, ) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 51aef9ef5..c2f3325c9 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -14,6 +14,8 @@ #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] use std::alloc::{GlobalAlloc, Layout}; +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] +use std::ptr::NonNull; #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))] #[derive(Default)] @@ -35,8 +37,10 @@ unsafe impl GlobalAlloc for MiMallocAllocator { } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - // SAFETY: ptr came from this allocator and layout.size() is the original allocation size. - unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) } + // SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's + // dealloc contract, and layout.size() is the original allocation size. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { diff --git a/rustfs/src/version.rs b/rustfs/src/version.rs index d94adedaf..7384b47a7 100644 --- a/rustfs/src/version.rs +++ b/rustfs/src/version.rs @@ -14,8 +14,6 @@ use const_str::concat; use shadow_rs::shadow; -use std::path::Path; -use std::process::Command; shadow!(build); @@ -47,10 +45,6 @@ pub const DISPLAY_VERSION: &str = { type VersionParseResult = Result<(u32, u32, u32, Option), Box>; -fn build_version_override() -> Option<&'static str> { - BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty()) -} - fn version_ref(version: &str) -> String { if version.starts_with("refs/tags/") || version.starts_with('@') { version.to_string() @@ -61,91 +55,7 @@ fn version_ref(version: &str) -> String { #[allow(clippy::const_is_empty)] pub fn get_version() -> String { - if let Some(version) = build_version_override() { - return version_ref(version); - } - - // Get the latest tag - if let Ok(latest_tag) = get_latest_tag() { - // Check if current commit is newer than the latest tag - if is_head_newer_than_tag(&latest_tag) { - // If current commit is newer, increment the version number - if let Ok(new_version) = increment_version(&latest_tag) { - return format!("refs/tags/{new_version}"); - } - } - - // If current commit is the latest tag, or version increment failed, return current tag - return format!("refs/tags/{latest_tag}"); - } - - // If no tag exists, use original logic - if !build::TAG.is_empty() { - format!("refs/tags/{}", build::TAG) - } else if !build::SHORT_COMMIT.is_empty() { - format!("@{}", build::SHORT_COMMIT) - } else { - format!("refs/tags/{}", build::PKG_VERSION) - } -} - -/// Get the latest git tag -fn get_latest_tag() -> Result> { - let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?; - - if output.status.success() { - let tag = String::from_utf8(output.stdout)?; - Ok(tag.trim().to_string()) - } else { - Err("Failed to get latest tag".into()) - } -} - -/// Check if current HEAD is newer than specified tag -fn is_head_newer_than_tag(tag: &str) -> bool { - is_head_newer_than_tag_in(Path::new("."), tag) -} - -fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool { - let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output(); - let tag_commit = Command::new("git") - .current_dir(repo) - .args(["rev-list", "-n", "1", tag]) - .output(); - - let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else { - return false; - }; - - if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout { - return false; - } - - let output = Command::new("git") - .current_dir(repo) - .args(["merge-base", "--is-ancestor", tag, "HEAD"]) - .output(); - - match output { - Ok(result) => result.status.success(), - Err(_) => false, - } -} - -/// Increment version number (increase patch version) -fn increment_version(version: &str) -> Result> { - // Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19")) - let (major, minor, patch, pre_release) = parse_version(version)?; - - // If there's a pre-release identifier, increment the pre-release version number - if let Some(pre) = pre_release - && let Some(new_pre) = increment_pre_release(&pre) - { - return Ok(format!("{major}.{minor}.{patch}-{new_pre}")); - } - - // Otherwise increment patch version number - Ok(format!("{major}.{minor}.{}", patch + 1)) + version_ref(DISPLAY_VERSION) } /// Parse version number @@ -166,28 +76,6 @@ pub fn parse_version(version: &str) -> VersionParseResult { Ok((major, minor, patch, pre_release)) } -/// Increment pre-release version number -fn increment_pre_release(pre_release: &str) -> Option { - // Handle pre-release versions like "alpha.19" - let parts: Vec<&str> = pre_release.split('.').collect(); - if parts.len() == 2 - && let Ok(num) = parts[1].parse::() - { - return Some(format!("{}.{}", parts[0], num + 1)); - } - - // Handle pre-release versions like "alpha19" - if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) { - let prefix = &pre_release[..=pos]; - let suffix = &pre_release[pos + 1..]; - if let Ok(num) = suffix.parse::() { - return Some(format!("{prefix}{}", num + 1)); - } - } - - None -} - /// Clean version string - removes common prefixes pub fn clean_version(version: &str) -> String { version @@ -284,34 +172,6 @@ mod tests { use super::*; use tracing::debug; - fn run_git(repo: &Path, args: &[&str]) { - let status = Command::new("git").current_dir(repo).args(args).status().unwrap(); - assert!(status.success(), "git command failed: git {}", args.join(" ")); - } - - #[test] - fn test_is_head_newer_than_tag_requires_strict_descendant() { - let repo = tempfile::tempdir().unwrap(); - run_git(repo.path(), &["init", "--quiet"]); - run_git(repo.path(), &["config", "user.name", "RustFS Tests"]); - run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]); - run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]); - run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]); - - assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3")); - - run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]); - - assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3")); - } - - #[test] - fn build_version_override_is_used_for_current_version_when_set() { - if let Some(version) = build_version_override() { - assert_eq!(get_version(), version_ref(version)); - } - } - #[test] fn version_ref_keeps_existing_ref_prefixes() { assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3"); @@ -319,6 +179,11 @@ mod tests { assert_eq!(version_ref("@abc123"), "@abc123"); } + #[test] + fn get_version_uses_build_metadata() { + assert_eq!(get_version(), version_ref(DISPLAY_VERSION)); + } + #[test] fn test_parse_version() { // Test standard version parsing @@ -336,27 +201,6 @@ mod tests { assert_eq!(pre_release, Some("alpha.19".to_string())); } - #[test] - fn test_increment_pre_release() { - // Test alpha.19 -> alpha.20 - assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string())); - - // Test beta.5 -> beta.6 - assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string())); - - // Test unparsable case - assert_eq!(increment_pre_release("unknown"), None); - } - - #[test] - fn test_increment_version() { - // Test pre-release version increment - assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20"); - - // Test standard version increment - assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1"); - } - #[test] fn test_version_format() { // Test if version format starts with refs/tags/