diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 8facca0e2..2cde94551 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34 -sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535 +sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json new file mode 100644 index 000000000..6cadc7815 --- /dev/null +++ b/.config/ecstore-required-tests.json @@ -0,0 +1,72 @@ +{ + "lane": "ci/test-and-lint", + "tests": [ + { + "invariant": "write-quorum", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one" + }, + { + "invariant": "metadata-rollback", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails" + }, + { + "invariant": "stale-writer", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss" + }, + { + "invariant": "range-body", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice" + }, + { + "invariant": "multipart-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup" + }, + { + "invariant": "list-uncommitted-version", + "suite": "rustfs-filemeta", + "name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge" + }, + { + "invariant": "minio-object-fixture", + "suite": "rustfs-filemeta", + "name": "filemeta::test::parses_real_minio_object_xlmeta" + }, + { + "invariant": "corrupt-part-arrays", + "suite": "rustfs-filemeta", + "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics" + } + ], + "fixtures": [ + { + "path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex", + "sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex", + "sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex", + "sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex", + "sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex", + "sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc00bf5f..98fc6cc30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} run: | mkdir -p artifacts/test-and-lint + rm -f target/nextest/ci/junit.xml ./scripts/ci/resource_sampler.sh start nextest trap './scripts/ci/resource_sampler.sh stop' EXIT set +e @@ -277,6 +278,12 @@ jobs: --status-level all --final-status-level all \ 2>&1 | tee artifacts/test-and-lint/nextest.log status=${PIPESTATUS[0]} + if [[ "${status}" -eq 0 ]]; then + cargo nextest list --profile ci --all --exclude e2e_test --message-format json \ + > artifacts/test-and-lint/core-test-listing.json \ + && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \ + && test -s target/nextest/ci/junit.xml || status=$? + fi { echo "command=cargo nextest run --profile ci --all --exclude e2e_test" echo "exit_status=${status}" diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index d822b4990..0c8c72cd0 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -18,7 +18,7 @@ on: workflow_dispatch: inputs: from_version: - description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)' + description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.' required: false default: '1.0.0-rc.3' from_url: @@ -26,7 +26,7 @@ on: required: false type: string to_version: - description: 'NEW RustFS release tag (leave empty for latest nightly)' + description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.' required: false to_url: description: 'NEW .deb URL. Overrides to_version / nightly default.' @@ -145,6 +145,7 @@ jobs: continue-on-error: true env: LOG_FILE: /tmp/rustfs-upgrade.log + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | set -euo pipefail chmod +x auto-testing/rustfs-upgrade-test.sh @@ -175,6 +176,29 @@ jobs: else ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi + # Fail fast with a clear message when a requested release tag has + # no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of + # letting the suite die mid-run on a 404. + check_release_asset() { + local version="$1" tag asset url + [ -n "${version}" ] && [ "${version}" != "null" ] || return 0 + tag="${version#v}" + asset="rustfs_${tag//-/.}_amd64.deb" + url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}" + if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then + echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2 + echo " ${url}" >&2 + echo "Pick a tag whose release ships a .deb (check its release assets)." >&2 + exit 1 + fi + echo "resolved ${tag} -> ${url}" + } + if [ -z "${FROM_URL}" ]; then + check_release_asset "${FROM_VERSION}" + fi + if [ -z "${TO_URL}" ]; then + check_release_asset "${TO_VERSION}" + fi ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}" - name: Generate report diff --git a/.gitignore b/.gitignore index b8d2eecbf..60325653d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ profile.json *.zst .secrets *.go +!crates/zip/tests/fixtures/snowball/**/generate/*.go *.pb *.svg deploy/logs/*.log.* diff --git a/Cargo.lock b/Cargo.lock index e9e7d9fe5..8206b1951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "amq-protocol" version = "10.6.3" @@ -330,6 +336,19 @@ dependencies = [ "rustversion", ] +[[package]] +name = "archive-trait" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6080ea14ccf9019d7ce572c319e581c20a805c9bdc6dbfb9b988da003cbd1a3" +dependencies = [ + "cap-std", + "thiserror 2.0.20", + "tokio", + "walkdir", + "windows-sys 0.60.2", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -506,7 +525,7 @@ dependencies = [ "arrow-select", "chrono", "half", - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "lexical-core", "memchr", @@ -809,7 +828,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -891,9 +910,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -958,9 +977,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -986,9 +1005,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.117.0" +version = "1.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b602641be84ebe5f96606cfefe4b96efaae1fd947c1b34ea8513b8ac0d2d8d" +checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7" dependencies = [ "arc-swap", "aws-credential-types", @@ -1012,9 +1031,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.144.0" +version = "1.145.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30dc8bf6baaf7d46336a0ca2c69f223d9b90d7a801fb3e28f7ea17b00dc6b1de" +checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93" dependencies = [ "arc-swap", "aws-credential-types", @@ -1049,9 +1068,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.108.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15301b04372832947916607983b114b3374b9db0be058a00fb7513800de1f05" +checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" dependencies = [ "arc-swap", "aws-credential-types", @@ -1075,9 +1094,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.110.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cc2c205cb27108183cf1856333f7d584c2ba0f505421b4209ca5828f9ea899" +checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" dependencies = [ "arc-swap", "aws-credential-types", @@ -1101,9 +1120,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.113.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68182ecb449f7537db0f4d5d25917789cf41e32074a9fe47b6a0b847fe1d2032" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ "arc-swap", "aws-credential-types", @@ -1235,7 +1254,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap 2.14.1", + "indexmap 2.14.2", "pin-project-lite", "rustls", "rustls-native-certs", @@ -1406,9 +1425,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1660,7 +1679,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1679,7 +1698,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1734,7 +1753,7 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1873,6 +1892,36 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cargo-platform" version = "0.3.3" @@ -1951,9 +2000,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -2061,7 +2110,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -2108,7 +2157,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2531,7 +2580,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -2556,11 +2605,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "typenum", ] @@ -2759,7 +2808,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2803,7 +2852,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2868,7 +2917,7 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "object_store", @@ -2943,7 +2992,7 @@ dependencies = [ "foldhash 0.2.0", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "libc", "log", @@ -3148,7 +3197,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "recursive", "serde_json", @@ -3163,7 +3212,7 @@ checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", ] @@ -3304,7 +3353,7 @@ checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -3319,7 +3368,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "recursive", @@ -3341,7 +3390,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "parking_lot", "petgraph 0.8.3", @@ -3375,7 +3424,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "parking_lot", "pin-project", @@ -3426,7 +3475,7 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.15.0", "log", "num-traits", @@ -3479,7 +3528,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap 2.14.1", + "indexmap 2.14.2", "log", "recursive", "regex", @@ -3852,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3929,7 +3978,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -4117,7 +4166,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.9", + "generic-array 0.14.7", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4341,9 +4390,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "findshlibs" @@ -4442,6 +4491,17 @@ dependencies = [ "pe-unwind-info", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4517,7 +4577,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -4562,9 +4622,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4577,7 +4637,7 @@ version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rustversion", "typenum", ] @@ -4656,7 +4716,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ "fnv", "hashbrown 0.16.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "stable_deref_trait", ] @@ -4934,7 +4994,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5583,9 +5643,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.1" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -5600,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -5626,6 +5686,28 @@ dependencies = [ "tempfile", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + [[package]] name = "io-uring" version = "0.7.14" @@ -5847,9 +5929,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -5885,7 +5967,7 @@ dependencies = [ "crc", "crc32c", "flate2", - "indexmap 2.14.1", + "indexmap 2.14.2", "lz4", "snap", "uuid", @@ -5918,7 +6000,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -6328,6 +6410,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md-5" version = "0.10.6" @@ -6398,7 +6486,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "metrics", "ordered-float 5.5.0", "quanta", @@ -7027,7 +7115,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -7703,7 +7791,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -7714,7 +7802,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.1", + "indexmap 2.14.2", "serde", ] @@ -7989,9 +8077,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -8095,7 +8183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -8943,7 +9031,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -9662,6 +9750,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", @@ -9956,7 +10045,7 @@ dependencies = [ "bytes", "fnv", "hmac 0.13.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "kafka-protocol", "metrics", "pbkdf2 0.13.0", @@ -10903,9 +10992,16 @@ dependencies = [ name = "rustfs-zip" version = "1.0.0-rc.5" dependencies = [ + "astral-tokio-tar", "async-compression", + "futures", "hotpath", "rustfs-rio", + "serde", + "serde_json", + "sha2 0.11.0", + "tar-codec", + "tar-framing", "thiserror 2.0.20", "tokio", ] @@ -10966,6 +11062,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.43" @@ -11293,7 +11399,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11405,7 +11511,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11424,7 +11530,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -11469,7 +11575,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11495,7 +11601,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.1", + "indexmap 2.14.2", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11549,7 +11655,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12146,9 +12252,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -12241,6 +12347,28 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar-codec" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea42eb144d30fcbf32c26dfea8959175bb8335bfb7b18d20c6ed32edecf0551" +dependencies = [ + "archive-trait", + "tar-framing", + "thiserror 2.0.20", + "tokio", +] + +[[package]] +name = "tar-framing" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783223a7a6590be4227cb821e7ce80372575511f67c824921aba0752d8ad5573" +dependencies = [ + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "tcp-stream" version = "0.34.14" @@ -12271,7 +12399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -12367,7 +12495,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12513,7 +12641,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -12558,9 +12686,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -12642,7 +12770,7 @@ version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "toml_datetime", "toml_parser", "winnow", @@ -12736,7 +12864,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.1", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -13239,9 +13367,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -13252,9 +13380,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -13262,9 +13390,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -13272,22 +13400,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -13307,9 +13435,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -13520,7 +13648,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -13538,14 +13675,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -13563,48 +13717,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.4" @@ -13614,6 +13816,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.1", + "windows-sys 0.52.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -13856,7 +14068,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -13873,7 +14085,7 @@ dependencies = [ "flate2", "getrandom 0.4.3", "hmac 0.13.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "lzma-rust2", "memchr", "pbkdf2 0.13.0", @@ -13921,18 +14133,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 05ddc1110..a7ecf889b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,7 +168,7 @@ reqwest = "0.13.4" rustfs-kafka-async = { version = "1.3.1" } socket2 = { version = "0.6.5" } tokio = { version = "1.53.1" } -tokio-rustls = { default-features = false, version = "0.26.4" } +tokio-rustls = { default-features = false, version = "0.26.5" } tokio-stream = { version = "0.1.19" } tokio-test = "0.4.5" tokio-util = { version = "0.7.19" } @@ -234,15 +234,19 @@ tokio-postgres-rustls = "0.14.0" # Utilities and Tools anyhow = "1.0.104" arc-swap = "1.9.2" -# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams. +# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork. astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" } +# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures. +tar-codec = "0.0.14" +tar-framing = "0.0.14" atoi = "3.1.0" atomic_enum = "0.3.0" -aws-config = { version = "1.11.0" } +aws-config = { version = "1.12.0" } aws-credential-types = { version = "1.3.0" } -aws-sdk-kms = { default-features = false, version = "1.117.0" } -aws-sdk-s3 = { default-features = false, version = "1.144.0" } -aws-sdk-sts = { default-features = false, version = "1.113.0" } +aws-sdk-kms = { default-features = false, version = "1.118.0" } +aws-sdk-s3 = { default-features = false, version = "1.145.0" } +aws-sdk-sts = { default-features = false, version = "1.114.0" } +aws-smithy-async = { version = "1.3.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-runtime-api = { version = "1.16.0" } aws-smithy-types = { version = "1.6.3" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 50aa7c1d3..d36c9d85b 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [ windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] } [dev-dependencies] +aws-smithy-async.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] } criterion = { workspace = true, features = ["html_reports"] } temp-env = { workspace = true, features = ["async_closure"] } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 7649e3ac9..dbb5dd154 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -584,33 +584,173 @@ impl ExpiryOp for FreeVersionTask { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TransitionDeleteVersionPlan { + Direct { version_id_exact: bool }, + ProbeLegacyUnknown, +} + +fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result { + use rustfs_utils::http::metadata_compat::{ + SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str, + }; + + if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) { + let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID); + if version_key_present { + if oi.transitioned_object.version_id.is_empty() { + let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| { + rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key) + .is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID)) + && !value.is_empty() + }); + if !has_non_empty_version { + // MinIO writes the transitioned-versionID key with an empty value + // for unversioned tier objects. The backend probe remains the proof. + return Ok(true); + } + } else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID) + == Some(oi.transitioned_object.version_id.as_str()) + { + return Ok(true); + } + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy remote tier version metadata is conflicting or malformed", + )); + } + if !oi.transitioned_object.version_id.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy remote tier version metadata is missing or inconsistent", + )); + } + return Ok(true); + } + let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object has conflicting transition version state metadata", + ) + })?; + if persisted != oi.transition_version_state.as_str() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object transition version state metadata changed during decoding", + )); + } + Ok(false) +} + +fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result { + match oi.transition_version_state { + rustfs_filemeta::TransitionVersionState::Unknown => { + if legacy_transition_version_state_missing(oi)? { + Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown) + } else { + validate_transition_remote_version(oi) + .map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }) + } + } + _ => validate_transition_remote_version(oi) + .map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ResolvedTransitionDeleteVersion { + version_id_exact: bool, + remote_already_missing: bool, +} + async fn acquire_free_version_tier_lease( oi: &ObjectInfo, tier_config_mgr: &Arc>, -) -> Result<(TierOperationLease, bool), std::io::Error> { - let version_id_exact = validate_transition_remote_version(oi)?; +) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> { + let delete_plan = transition_remote_version_delete_plan(oi)?; let identity = tier_destination_id_from_metadata(&oi.user_defined)? .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?; let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity) .await .map_err(std::io::Error::other)?; - Ok((lease, version_id_exact)) + Ok((lease, delete_plan)) +} + +async fn resolve_transition_delete_version_plan( + oi: &ObjectInfo, + lease: &TierOperationLease, + delete_plan: TransitionDeleteVersionPlan, +) -> Result { + match delete_plan { + TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion { + version_id_exact, + remote_already_missing: false, + }), + TransitionDeleteVersionPlan::ProbeLegacyUnknown => { + let expected_version = oi.transitioned_object.version_id.as_str(); + if expected_version.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "remote tier cannot safely delete a legacy object without an exact version ID", + )); + } + let probe = lease + .probe_transition_version(&oi.transitioned_object.name, expected_version) + .await?; + match (expected_version, probe) { + (expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual)) + if expected == actual => + { + lease.validate_remote_version_id(expected)?; + Ok(ResolvedTransitionDeleteVersion { + version_id_exact: true, + remote_already_missing: false, + }) + } + (_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => { + Ok(ResolvedTransitionDeleteVersion { + version_id_exact: false, + remote_already_missing: true, + }) + } + (_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot prove legacy transition delete state", + )), + _ => Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "remote tier object version state is unknown", + )), + } + } + } +} + +async fn execute_resolved_transition_delete( + oi: &ObjectInfo, + lease: &TierOperationLease, + resolved: ResolvedTransitionDeleteVersion, +) -> Result<(), std::io::Error> { + if !resolved.remote_already_missing { + delete_object_from_remote_tier_with_lease_idempotent( + &oi.transitioned_object.name, + &oi.transitioned_object.version_id, + lease, + resolved.version_id_exact, + ) + .await?; + } + Ok(()) } async fn delete_free_version_remote_object_with_lease( oi: &ObjectInfo, lease: &TierOperationLease, - version_id_exact: bool, + delete_plan: TransitionDeleteVersionPlan, ) -> Result<(), std::io::Error> { - delete_object_from_remote_tier_with_lease_idempotent( - &oi.transitioned_object.name, - &oi.transitioned_object.version_id, - lease, - version_id_exact, - ) - .await?; - Ok(()) + let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?; + execute_resolved_transition_delete(oi, lease, resolved).await } fn free_version_physical_topology_generation(api: &ECStore) -> String { @@ -641,6 +781,16 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown { + let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?; + let expected_legacy_missing = legacy_transition_version_state_missing(expected)?; + if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + && expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + && candidate_legacy_missing + && expected_legacy_missing + && candidate.transitioned_object.version_id == expected.transitioned_object.version_id + { + return Ok(true); + } return Err(std::io::Error::new( std::io::ErrorKind::WouldBlock, "tier free-version remote version state is unknown", @@ -716,7 +866,7 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: .acquire_bucket_lifecycle_read_lock(&oi.bucket) .await .map_err(std::io::Error::other)?; - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?; let local_object = encode_dir_object(&oi.name); let object_guards = api .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object) @@ -734,16 +884,30 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: "tier free-version cleanup fence is invalid before remote delete", )); } + let resolved = tokio::select! { + _ = cancel.cancelled() => { + return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); + } + result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => { + result.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out") + })?? + } + }; + if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "tier free-version cleanup fence changed after remote probe", + )); + } tokio::select! { _ = cancel.cancelled() => { return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled")); } - result = tokio::time::timeout_at( - deadline, - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact), - ) => { - result - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??; + result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => { + result.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out") + })??; } } if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { @@ -791,8 +955,8 @@ async fn delete_free_version_remote_object( oi: &ObjectInfo, tier_config_mgr: &Arc>, ) -> Result<(), std::io::Error> { - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await } #[allow( @@ -808,8 +972,8 @@ where F: FnOnce() -> Fut, Fut: std::future::Future, { - let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; - delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?; + let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?; + delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?; let result = delete_local().await; drop(lease); Ok(result) @@ -4688,6 +4852,39 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result Result { + let version = oi.transitioned_object.version_id.as_str(); + match oi.transition_version_state { + rustfs_filemeta::TransitionVersionState::Unknown => { + if !legacy_transition_version_state_missing(oi)? { + return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct); + } + if version.is_empty() { + Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned) + } else { + Ok(TransitionReadVersionPlan::Direct) + } + } + rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct), + rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct), + rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => { + Ok(TransitionReadVersionPlan::Direct) + } + _ => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object version state conflicts with its version ID", + )), + } +} + // The resolver joins the tier manager as the second injected port this read // needs; grouping the request half into a struct would churn every call site of // a bug fix. @@ -4702,7 +4899,12 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( tier_config_mgr: &Arc>, resolver: Option<&dyn ObjectEncryptionResolver>, ) -> Result { - validate_transition_remote_version(oi)?; + let read_plan = transition_remote_version_read_plan(oi)?; + // Reject invalid ranges and encryption requests before a compatibility + // probe can amplify them into remote listing work. + let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver) + .await + .map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?; let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; let lease = match expected_identity { Some(identity) => { @@ -4716,7 +4918,36 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( Err(err) => return Err(std::io::Error::other(err)), }; - tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; + match read_plan { + TransitionReadVersionPlan::Direct => { + tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; + } + TransitionReadVersionPlan::ProbeLegacyUnversioned => { + // RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing + // after an admin reconcile can persist every proven legacy state. + let probe = tokio::time::timeout( + LEGACY_TRANSITION_READ_PROBE_TIMEOUT, + tgt_client.probe_transition_candidate(&oi.transitioned_object.name), + ) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??; + match probe { + crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {} + crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => { + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "remote tier cannot prove legacy unversioned transition state", + )); + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "remote tier object version state is unknown", + )); + } + } + } + } // The same read plan the local path uses, so the tier fetch is positioned in // the object's *stored* coordinate system and the stream is handed the same @@ -4724,9 +4955,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( // through a plaintext-coordinate range and skipping the transform is how a // transitioned SSE object used to come back as silently corrupt bytes of the // right length (rustfs/rustfs#6025). - let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver) - .await - .map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?; let (off, length) = (plan.storage_offset() as i64, plan.storage_length()); let mut gopts = WarmBackendGetOpts::default(); @@ -5599,11 +5827,13 @@ mod tests { use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader}; #[cfg(feature = "test-util")] + use crate::services::tier::test_util::MockWarmOp; + #[cfg(feature = "test-util")] use crate::services::tier::test_util::register_mock_tier; #[cfg(feature = "test-util")] use crate::services::tier::tier::TierConfigMgr; #[cfg(feature = "test-util")] - use crate::services::tier::warm_backend::WarmBackend as _; + use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _}; use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; @@ -6299,7 +6529,75 @@ mod tests { #[cfg(feature = "test-util")] #[tokio::test] - async fn transitioned_get_rejects_unknown_version_state_before_backend_io() { + async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy transitioned object body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body.clone()), + i64::try_from(body.len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + let mut user_defined = HashMap::new(); + insert_legacy_transition_version_id(&mut user_defined, &remote_version); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: i64::try_from(body.len()).expect("body length should fit"), + transitioned_object: TransitionedObject { + name: remote_object, + version_id: remote_version, + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec { + is_suffix_length: false, + start: 7, + end: 18, + }); + let mut reader = get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &range, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + .expect("legacy unknown state should still allow a non-destructive read"); + let mut got = Vec::new(); + reader + .stream + .read_to_end(&mut got) + .await + .expect("transitioned reader should drain"); + + assert_eq!(got, &body.as_ref()[7..=18]); + assert_eq!(backend.get_count().await, 1); + assert_eq!(backend.remove_count().await, 0); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier).await, + 0, + "tier generation lease should release after EOF" + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() { let manager = TierConfigMgr::new(); let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&manager, &tier).await; @@ -6315,6 +6613,181 @@ mod tests { ..Default::default() }, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(), + ..Default::default() + }; + + let err = match get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + { + Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"), + Err(err) => err, + }; + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(backend.op_log().await, Vec::::new()); + assert_eq!(backend.get_count().await, 0); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + + for persisted_version in [ + Uuid::nil().to_string(), + "\u{fffd}".to_string(), + "bad\u{0001}version".to_string(), + ] { + let mut user_defined = HashMap::new(); + insert_legacy_transition_version_id(&mut user_defined, &persisted_version); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + transitioned_object: TransitionedObject { + name: "remote/object".to_string(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = match get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + { + Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"), + Err(err) => err, + }; + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + assert_eq!(backend.op_log().await, Vec::::new()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + backend.set_put_remote_version(Some(String::new())).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy unversioned transitioned object body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body.clone()), + i64::try_from(body.len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + assert!(remote_version.is_empty()); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: i64::try_from(body.len()).expect("body length should fit"), + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier: tier.clone(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(), + ..Default::default() + }; + + let mut reader = get_transitioned_object_reader_with_tier_manager( + &object_info.bucket, + &object_info.name, + &None, + &HeaderMap::new(), + &object_info, + &ObjectOptions::default(), + &manager, + None, + ) + .await + .expect("probe-proven legacy unversioned state should allow a non-destructive read"); + let mut got = Vec::new(); + reader + .stream + .read_to_end(&mut got) + .await + .expect("transitioned reader should drain"); + + assert_eq!(got, body.as_ref()); + assert_eq!(backend.remove_count().await, 0); + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Put { + object: remote_object.clone() + }, + MockWarmOp::Probe { + object: remote_object.clone() + }, + MockWarmOp::Get { object: remote_object }, + ] + ); + assert_eq!( + TierConfigMgr::active_operation_lease_count(&manager, &tier).await, + 0, + "tier generation lease should release after EOF" + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() { + let manager = TierConfigMgr::new(); + let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&manager, &tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent( + "versioned-candidate".to_string(), + ))) + .await; + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(), + tier, + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, ..Default::default() }; @@ -6330,19 +6803,28 @@ mod tests { ) .await { - Ok(_) => panic!("unknown remote version state must fail before backend IO"), + Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"), Err(err) => err, }; assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]); assert_eq!(backend.get_count().await, 0); + assert_eq!(backend.remove_count().await, 0); } #[cfg(feature = "test-util")] #[tokio::test] - async fn free_version_delete_rejects_unknown_version_state_before_backend_io() { + async fn free_version_delete_rejects_explicit_unknown_before_backend_io() { let manager = TierConfigMgr::new(); let backend = register_mock_tier(&manager, "WARM").await; + let identity = test_tier_destination_identity(&manager, "WARM").await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(), + ); let object_info = ObjectInfo { transitioned_object: TransitionedObject { name: "remote/object".to_string(), @@ -6351,17 +6833,251 @@ mod tests { ..Default::default() }, transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), ..Default::default() }; let err = super::delete_free_version_remote_object(&object_info, &manager) .await - .expect_err("unknown remote version state must fail before backend IO"); + .expect_err("explicit unknown cleanup must fail before backend IO"); assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("version state is unknown")); + assert_eq!(backend.op_log().await, Vec::::new()); assert_eq!(backend.remove_count().await, 0); } + #[cfg(feature = "test-util")] + async fn test_tier_destination_identity( + manager: &Arc>, + tier: &str, + ) -> crate::services::tier::tier::TierDestinationId { + TierConfigMgr::acquire_operation_lease(manager, tier) + .await + .expect("test tier lease should be available") + .backend_identity() + } + + #[cfg(feature = "test-util")] + fn user_defined_with_tier_destination_identity( + identity: crate::services::tier::tier::TierDestinationId, + ) -> HashMap { + let mut user_defined = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(identity), + ); + user_defined + } + + #[cfg(feature = "test-util")] + fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap { + let mut user_defined = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + state.as_str().to_string(), + ); + user_defined + } + + #[cfg(feature = "test-util")] + fn insert_legacy_transition_version_id(user_defined: &mut HashMap, version_id: &str) { + rustfs_utils::http::metadata_compat::insert_str( + user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID, + version_id.to_string(), + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() { + let manager = TierConfigMgr::new(); + register_mock_tier(&manager, "WARM").await; + let identity = test_tier_destination_identity(&manager, "WARM").await; + let mut legacy_metadata = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version"); + let mut explicit_metadata = legacy_metadata.clone(); + rustfs_utils::http::metadata_compat::insert_str( + &mut explicit_metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(), + ); + let make_info = |user_defined: HashMap| ObjectInfo { + transitioned_object: TransitionedObject { + name: "remote/object".to_string(), + version_id: "legacy-version".to_string(), + tier: "WARM".to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata)) + .expect_err("mixed legacy-missing and explicit unknown provenance must fail closed"); + + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy exact cleanup body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body), + i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + let mut user_defined = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut user_defined, &remote_version); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing)) + .await; + assert_eq!( + backend + .probe_transition_candidate_state(&remote_object) + .await + .expect("current remote view should be readable"), + TransitionCandidateProbe::Missing, + "a current delete marker must hide the historical data version from an unversioned probe" + ); + backend.clear_op_log().await; + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: remote_version, + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("probe-proven legacy exact cleanup should delete the remote version"); + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("a retry after the exact remote version is already missing should be idempotent"); + + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Get { + object: remote_object.clone() + }, + MockWarmOp::Remove { + object: remote_object.clone() + }, + MockWarmOp::Get { + object: remote_object.clone() + }, + ] + ); + assert_eq!( + backend.remove_versions().await, + vec![(remote_object, object_info.transitioned_object.version_id)] + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_retains_legacy_unknown_unversioned_object() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + backend.set_put_remote_version(Some(String::new())).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + let body = Bytes::from_static(b"legacy unversioned cleanup body"); + let remote_version = backend + .put( + &remote_object, + ReaderImpl::Body(body), + i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"), + ) + .await + .expect("mock remote object should be stored"); + assert!(remote_version.is_empty()); + backend.clear_op_log().await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new()); + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: String::new(), + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + let err = super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect_err("legacy unversioned cleanup cannot exclude a versioning-state race"); + + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + assert!(backend.op_log().await.is_empty()); + assert_eq!(backend.remove_count().await, 0); + assert!(backend.remove_versions().await.is_empty()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_delete_does_not_remove_a_different_remote_version() { + let manager = TierConfigMgr::new(); + let tier = "WARM"; + let backend = register_mock_tier(&manager, tier).await; + let identity = test_tier_destination_identity(&manager, tier).await; + let remote_object = format!("remote/{}", Uuid::new_v4()); + backend.set_put_remote_version(Some("different-version".to_string())).await; + backend + .put( + &remote_object, + ReaderImpl::Body(Bytes::from_static(b"different remote version")), + i64::try_from(b"different remote version".len()).expect("body length should fit"), + ) + .await + .expect("different remote version should be stored"); + backend.clear_op_log().await; + let mut user_defined = user_defined_with_tier_destination_identity(identity); + insert_legacy_transition_version_id(&mut user_defined, "legacy-version"); + let object_info = ObjectInfo { + transitioned_object: TransitionedObject { + name: remote_object.clone(), + version_id: "legacy-version".to_string(), + tier: tier.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown, + user_defined: user_defined.into(), + ..Default::default() + }; + + super::delete_free_version_remote_object(&object_info, &manager) + .await + .expect("a missing exact legacy version should be an idempotent cleanup success"); + + assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]); + assert_eq!(backend.remove_count().await, 0); + assert!(backend.remove_versions().await.is_empty()); + } + #[cfg(feature = "test-util")] #[tokio::test] async fn free_version_remote_delete_requires_persisted_destination_identity() { diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index b480ba468..050228b25 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() @@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 3a6a7e451..2270bce3b 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match( let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(HTTPPreconditions { if_match: Some(observed_etag), @@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag.to_string()), ..Default::default() @@ -3780,6 +3783,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -3869,6 +3873,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() @@ -3893,6 +3898,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 6518451ff..ad2241629 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use super::runtime_boundary as runtime_sources; use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; @@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock> = LazyLock::n }); #[cfg(test)] -static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock< - std::sync::Mutex std::io::Result<()> + Send + Sync>>>, -> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None)); +type RemoteTierDeleteTestHook = Box std::io::Result<()> + Send + Sync>; + +#[cfg(test)] +static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(None)); #[derive(Debug)] struct RemoteDeleteBreaker { @@ -107,7 +107,7 @@ impl RemoteDeleteBreaker { fn prune(&mut self, now: Instant) { while let Some(ts) = self.failures.front().copied() { if now.duration_since(ts) > self.window { - self.failures.pop_front(); + let _ = self.failures.pop_front(); } else { break; } @@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool { return false; } - if let Some(source) = err.get_ref() { - if error_chain_contains_signer_header_marker(source) { - return true; - } + if let Some(source) = err.get_ref() + && error_chain_contains_signer_header_marker(source) + { + return true; } let message = err.to_string().to_ascii_lowercase(); @@ -205,7 +205,7 @@ impl ObjSweeper { #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_version(&mut self, vid: Option) -> &Self { - self.version_id = vid.clone(); + self.version_id = vid; self } @@ -219,7 +219,7 @@ impl ObjSweeper { #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn get_opts(&self) -> lifecycle::ObjectOpts { let mut opts = ObjectOpts { - version_id: self.version_id.clone(), + version_id: self.version_id, versioned: self.versioned, version_suspended: self.suspended, ..Default::default() @@ -388,8 +388,8 @@ impl Jentry { impl ExpiryOp for Jentry { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.tier_name).as_bytes()); - hasher.update(format!("{}", self.obj_name).as_bytes()); + hasher.update(self.tier_name.as_bytes()); + hasher.update(self.obj_name.as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager( tier_name: &str, tier_config_mgr: &Arc>, ) -> Result<(), std::io::Error> { - let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name) + let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name) .await .map_err(std::io::Error::other)?; delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 87df9fe0d..82e32f598 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs index 6768c9717..ddcbce8da 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs @@ -684,6 +684,7 @@ async fn write_checkpoint( }; let opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(preconditions), ..Default::default() }; diff --git a/crates/ecstore/src/bucket/on_demand_migration/breaker.rs b/crates/ecstore/src/bucket/on_demand_migration/breaker.rs index c41305f1d..46d25fca2 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/breaker.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/breaker.rs @@ -86,7 +86,12 @@ impl BreakerVerdict { Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => { BreakerVerdict::Failure } - Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral, + Some( + SourceError::AccessDenied + | SourceError::Unsupported(_) + | SourceError::InvalidPagination(_) + | SourceError::Other(_), + ) => BreakerVerdict::Neutral, } } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs index 8ff71196a..2720c7718 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs @@ -189,8 +189,8 @@ pub enum SourceListPlan { /// delimiter — the source's own roll-up boundary matches the request's. Page { prefix: String }, /// `filter.prefix` reaches past a delimiter, so every key the source could - /// contribute rolls into this one common prefix. One bounded probe listing - /// decides whether it exists; there is nothing to paginate. + /// contribute rolls into this one common prefix. Bounded probes follow + /// empty progressing pages until a key proves existence or the source ends. Folded { probe_prefix: String, common_prefix: String }, } @@ -279,6 +279,29 @@ pub struct FetchRequest { pub token: Option, } +/// Invalid pagination metadata. Opaque cursor values are never included in errors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ListPageError { + #[error("truncated listing has no continuation token")] + Missing, + #[error("truncated listing has an empty continuation token")] + Empty, + #[error("truncated listing repeats a continuation token")] + Repeated, +} + +pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> { + if is_truncated { + match next_token { + None => return Err(ListPageError::Missing), + Some("") => return Err(ListPageError::Empty), + Some(next) if Some(next) == token => return Err(ListPageError::Repeated), + Some(_) => {} + } + } + Ok(()) +} + #[derive(Debug, Default)] struct SideState { start: SideCursor, @@ -364,6 +387,11 @@ impl ListThroughMerger { /// or `filter.prefix` excludes it. pub fn disable_source(&mut self) { self.source.disabled = true; + // A refill can fail after a valid first page. A local-only response + // must discard both that source payload and its ordering horizon. + self.source.entries.clear(); + self.source.pages.clear(); + self.source.more = false; } pub fn next_fetch(&self) -> Option { @@ -378,7 +406,13 @@ impl ListThroughMerger { /// Records one fetched page. `entries` must be sorted by `name` and already /// filtered with [`Self::accepts`]; the caller keeps the matching payloads /// in the same order. - pub fn push_page(&mut self, side: MergeSide, entries: Vec, is_truncated: bool, next_token: Option) { + pub fn push_page( + &mut self, + side: MergeSide, + entries: Vec, + is_truncated: bool, + next_token: Option, + ) -> Result<(), ListPageError> { let state = match side { MergeSide::Local => &mut self.local, MergeSide::Source => &mut self.source, @@ -387,15 +421,19 @@ impl ListThroughMerger { Some(last) => last.next_token.clone(), None => state.start.token.clone(), }; - // A truncated page without a cursor cannot be continued; treating the - // side as finished is the only alternative to looping on it forever. - state.more = is_truncated && next_token.is_some(); + validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?; + // Also reject a cycle through an earlier page in this bounded fetch. + if is_truncated && state.pages.iter().any(|page| page.token == next_token) { + return Err(ListPageError::Repeated); + } + state.more = is_truncated; state.pages.push(FetchedPage { token, count: entries.len(), next_token: is_truncated.then_some(next_token).flatten(), }); state.entries.extend(entries); + Ok(()) } pub fn finish(self) -> MergeOutcome { @@ -599,9 +637,15 @@ mod tests { let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys); let kept: Vec = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect(); buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned()); - merger.push_page(fetch.side, kept, truncated, next); + merger + .push_page(fetch.side, kept, truncated, next) + .expect("reference provider pages must advance"); } let outcome = merger.finish(); + assert_eq!(outcome.is_truncated, outcome.next_token.is_some()); + if outcome.is_truncated { + assert_ne!(outcome.next_token, token, "every truncated merged page must make progress"); + } page_sizes.push(outcome.picks.len()); for pick in &outcome.picks { let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone(); @@ -616,11 +660,25 @@ mod tests { } fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec { - let mut all: Vec = local.iter().chain(source.iter()).cloned().collect(); - all.sort(); - all.dedup(); - let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX); - entries + // This oracle builds the complete namespace independently of the + // provider's page/marker helper and the production merger. + let mut namespace = std::collections::BTreeMap::new(); + for key in local.iter().chain(source) { + let Some(suffix) = key.strip_prefix(prefix) else { + continue; + }; + if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty()) + && let Some((directory, _)) = suffix.split_once(delimiter) + { + namespace.insert(format!("{prefix}{directory}{delimiter}"), true); + continue; + } + namespace.insert(key.clone(), false); + } + namespace + .into_iter() + .map(|(name, is_prefix)| ListEntryKey { name, is_prefix }) + .collect() } #[test] @@ -662,7 +720,9 @@ mod tests { token: None }) ); - merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None) + .expect("local EOF is valid"); assert_eq!(merger.next_fetch(), None); let outcome = merger.finish(); assert_eq!(outcome.picks.len(), 1); @@ -683,12 +743,14 @@ mod tests { }; let mut merger = ListThroughMerger::new(1, Some(&resume)); merger.disable_source(); - merger.push_page( - MergeSide::Local, - vec![ListEntryKey::object("b"), ListEntryKey::object("c")], - true, - Some("local-2".to_string()), - ); + merger + .push_page( + MergeSide::Local, + vec![ListEntryKey::object("b"), ListEntryKey::object("c")], + true, + Some("local-2".to_string()), + ) + .expect("local cursor advances"); let outcome = merger.finish(); assert!(outcome.is_truncated); let token = outcome.next_token.expect("truncated page carries a token"); @@ -698,6 +760,212 @@ mod tests { assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed"); } + #[test] + fn truncated_pages_require_a_nonempty_advancing_cursor() { + for side in [MergeSide::Local, MergeSide::Source] { + for entries in [vec![], vec![ListEntryKey::object("a")]] { + for (next, expected) in [ + (None, Err(ListPageError::Missing)), + (Some(""), Err(ListPageError::Empty)), + (Some("stuck"), Err(ListPageError::Repeated)), + (Some("advances"), Ok(())), + ] { + let resume = ListThroughToken::new( + SideCursor { + token: Some("stuck".into()), + done: false, + }, + SideCursor { + token: Some("stuck".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string)); + assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}"); + let state = if side == MergeSide::Local { + &merger.local + } else { + &merger.source + }; + assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted"); + } + } + } + } + + #[test] + fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() { + let resume = ListThroughToken::new( + SideCursor { token: None, done: true }, + SideCursor { + token: Some("stuck".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + assert_eq!( + merger.next_fetch(), + Some(FetchRequest { + side: MergeSide::Source, + token: Some("stuck".into()) + }) + ); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())), + Err(ListPageError::Repeated) + ); + } + + #[test] + fn empty_pages_may_advance_within_the_fetch_budget_until_eof() { + let mut merger = ListThroughMerger::new(2, None); + merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF"); + for next in ["opaque-z", "opaque-a"] { + assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source); + merger + .push_page(MergeSide::Source, vec![], true, Some(next.into())) + .expect("opaque cursor advances regardless of sort order"); + } + assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget"); + let outcome = merger.finish(); + assert!(outcome.picks.is_empty()); + assert!(outcome.is_truncated); + let token = outcome.next_token.expect("empty progressing page has a cursor"); + assert_eq!(token.source.as_deref(), Some("opaque-a")); + let mut merger = ListThroughMerger::new(2, Some(&token)); + assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a")); + merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None) + .expect("source EOF"); + let outcome = merger.finish(); + assert_eq!( + outcome.picks, + vec![MergePick { + side: MergeSide::Source, + index: 0 + }] + ); + assert!(!outcome.is_truncated); + assert!(outcome.next_token.is_none()); + } + + #[test] + fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() { + let resume = ListThroughToken::new( + SideCursor { token: None, done: true }, + SideCursor { + token: Some("first".into()), + done: false, + }, + None, + ); + let mut merger = ListThroughMerger::new(2, Some(&resume)); + merger + .push_page(MergeSide::Source, vec![], true, Some("second".into())) + .expect("first page advances"); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("first".into())), + Err(ListPageError::Repeated) + ); + } + + #[test] + fn source_refill_failure_discards_buffered_source_entries_and_horizon() { + let mut merger = ListThroughMerger::new(2, None); + merger + .push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None) + .expect("local EOF"); + merger + .push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into())) + .expect("first source page advances"); + assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck")); + assert_eq!( + merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())), + Err(ListPageError::Repeated) + ); + merger.disable_source(); + let outcome = merger.finish(); + assert_eq!( + outcome.picks, + vec![MergePick { + side: MergeSide::Local, + index: 0 + }] + ); + assert!(!outcome.is_truncated); + assert!(outcome.next_token.is_none()); + } + + #[test] + fn list_through_static_namespace_boundary_matrix() { + let corpus = [ + "a", + "a/", + "a/b", + "a/b/child", + "a0", + "b", + "b/leaf", + "quote\"&<", + "space key", + "z", + "é", + "中/文", + ]; + for count in [0, 1, 3, 4, corpus.len()] { + let keys: Vec = corpus[..count].iter().map(|key| (*key).to_string()).collect(); + for placement in 0..3 { + let (local, source): (Vec<_>, Vec<_>) = + keys.iter() + .enumerate() + .fold((vec![], vec![]), |(mut local, mut source), (index, key)| { + if placement != 1 || index % 2 == 0 { + local.push(key.clone()); + } + if placement != 0 || index % 2 == 0 { + source.push(key.clone()); + } + (local, source) + }); + for prefix in ["", "a", "a/", "中/"] { + for delimiter in [None, Some("/")] { + for max_keys in [1, 3, 4] { + let oracle = expected(&local, &source, prefix, delimiter); + let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys); + assert_eq!( + emitted.iter().map(|(entry, _)| entry.clone()).collect::>(), + oracle, + "count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}" + ); + let expected_sizes: Vec<_> = if oracle.is_empty() { + vec![0] + } else { + oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect() + }; + assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree"); + } + } + } + } + } + } + + #[test] + fn list_through_large_overlap_walk_keeps_all_5300_keys() { + let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect(); + let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect(); + let (emitted, sizes) = walk(&local, &source, "", None, 333); + assert_eq!(emitted.len(), 5300); + for (index, (entry, side)) in emitted.iter().enumerate() { + assert_eq!(entry.name, format!("k{index:05}")); + assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source }); + } + assert_eq!(sizes, [vec![333; 15], vec![305]].concat()); + } + #[test] fn token_round_trips_and_rejects_tampering() { let token = ListThroughToken::new( @@ -796,7 +1064,10 @@ mod tests { } proptest! { - #![proptest_config(ProptestConfig::with_cases(256))] + #![proptest_config(ProptestConfig { + rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706), + ..ProptestConfig::with_cases(256) + })] /// Full pagination of a merged listing equals the sorted, deduplicated /// union of both sides, with every shared key served by local, and no diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 4782e05f0..f06301036 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -25,6 +25,7 @@ //! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never //! forwarded: v1 rejects SSE-C source objects outright. +use super::list_through::{ListPageError, validate_list_page}; use crate::bucket::remote_s3_client::{ PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config, }; @@ -223,6 +224,8 @@ pub enum SourceError { ServerError(u16), #[error("unsupported source object: {0}")] Unsupported(String), + #[error("invalid source listing: {0}")] + InvalidPagination(#[from] ListPageError), #[error("source request failed: {0}")] Other(String), } @@ -245,6 +248,7 @@ impl SourceError { SourceError::Connect(_) => "connect", SourceError::ServerError(_) => "server_error", SourceError::Unsupported(_) => "unsupported", + SourceError::InvalidPagination(_) => "invalid_pagination", SourceError::Other(_) => "other", } } @@ -714,6 +718,7 @@ impl SourceClient { ..*request }) .await?; + validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?; page.objects = page .objects .into_iter() @@ -800,11 +805,6 @@ impl SourceBackend for S3SourceBackend { let is_truncated = output.is_truncated.unwrap_or(false); let next_continuation_token = output.next_continuation_token; - if is_truncated && next_continuation_token.is_none() { - return Err(SourceError::Other( - "source reported a truncated listing without a continuation token".to_string(), - )); - } let objects = output .contents .unwrap_or_default() @@ -1274,7 +1274,9 @@ mod tests { data/photos/ outside/ "#; - let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await; + let next_body = body.replace("data/opaque", "data/next"); + let (client, requests) = + scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await; let first = client .list_page(&SourceListRequest { prefix: Some("photos/"), @@ -1336,7 +1338,104 @@ mod tests { .list_objects_v2(None, None, 10) .await .expect_err("truncated page without token is corrupt"); - assert!(matches!(err, SourceError::Other(_)), "{err:?}"); + assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}"); + } + + #[tokio::test] + async fn list_page_validates_s3_cursor_progress_before_mapping_entries() { + for contents in ["", "data/a1"] { + for (truncated, next, expected) in [ + (true, None, Some(ListPageError::Missing)), + (true, Some(""), Some(ListPageError::Empty)), + (true, Some("stuck"), Some(ListPageError::Repeated)), + (true, Some("opaque-next"), None), + (false, None, None), + (false, Some("stuck"), None), + ] { + let next_xml = next + .map(|next| format!("{next}")) + .unwrap_or_default(); + let body = format!( + "{truncated}{next_xml}{contents}" + ); + let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await; + let result = client + .list_page(&SourceListRequest { + continuation_token: Some("stuck"), + max_keys: 2, + ..Default::default() + }) + .await; + match expected { + Some(expected) => { + let error = result.expect_err("malformed pagination must fail at the provider boundary"); + assert!( + matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected), + "{error:?}" + ); + assert_eq!(error.class_label(), "invalid_pagination"); + assert!(!error.is_retryable()); + assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens"); + } + None => { + let page = result.expect("progressing empty/nonempty pages and EOF are valid"); + assert_eq!(page.is_truncated, truncated); + assert_eq!(page.next_continuation_token.as_deref(), next); + assert_eq!(page.objects.len(), usize::from(!contents.is_empty())); + if let Some(object) = page.objects.first() { + assert_eq!(object.key, "a"); + } + } + } + let requests = recorded(&requests); + assert_eq!(requests.len(), 1, "invalid pagination must not be retried"); + assert!(requests[0].uri.contains("continuation-token=stuck")); + } + } + } + + struct ListOnlyBackend(SourcePage); + + #[async_trait::async_trait] + impl SourceBackend for ListOnlyBackend { + async fn list(&self, request: &SourceListRequest<'_>) -> Result { + assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged"); + Ok(self.0.clone()) + } + + async fn head(&self, _key: &str) -> Result { + panic!("unexpected HEAD in list test") + } + async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result { + panic!("unexpected GET in list test") + } + async fn tagging(&self, _key: &str) -> Result, SourceError> { + panic!("unexpected tagging in list test") + } + async fn probe(&self) -> Result<(), SourceError> { + panic!("unexpected probe in list test") + } + } + + #[tokio::test] + async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() { + for (next, expected) in [ + (None, ListPageError::Missing), + (Some(""), ListPageError::Empty), + (Some("stuck"), ListPageError::Repeated), + ] { + let mut client = prefix_client(Some("data/".into())); + client.backend = Box::new(ListOnlyBackend(SourcePage { + is_truncated: true, + next_continuation_token: next.map(str::to_string), + ..Default::default() + })); + let error = client + .list_objects_v2(None, Some("stuck"), 2) + .await + .expect_err("all providers must advance pagination"); + assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected)); + } } const TAGGING_BODY: &str = r#" diff --git a/crates/ecstore/src/bucket/on_demand_migration/stats.rs b/crates/ecstore/src/bucket/on_demand_migration/stats.rs index ed3a7de2c..a483abde0 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/stats.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/stats.rs @@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason { SourceError::Connect(_) => PullFailureReason::SourceConnect, SourceError::ServerError(_) => PullFailureReason::SourceServerError, SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported, - SourceError::Other(_) => PullFailureReason::SourceOther, + SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther, } } } diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs index 20d3d0365..a434e09ac 100644 --- a/crates/ecstore/src/bucket/remote_s3_client.rs +++ b/crates/ecstore/src/bucket/remote_s3_client.rs @@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option { #[cfg(test)] mod tests { use super::*; + use aws_smithy_async::time::TimeSource; use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode; use std::sync::Mutex; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec { RemoteS3EndpointSpec { @@ -824,6 +825,174 @@ mod tests { ); } + #[derive(Clone, Debug)] + struct ClockSkewTimeSource(Arc); + + impl TimeSource for ClockSkewTimeSource { + fn now(&self) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst)) + } + } + + #[derive(Clone, Debug)] + struct ClockSkewConnector { + request_headers: RecordedHeaders, + error_code: &'static str, + skew_seconds: i64, + clock: ClockSkewTimeSource, + } + + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + .unwrap_or_else(|| panic!("signed request must contain {name}")) + } + + fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime { + chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ") + .expect("SDK signing timestamp must use the SigV4 format") + } + + impl SmithyHttpConnector for ClockSkewConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let mut headers = self.request_headers.lock().expect("clock skew request capture lock"); + assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD"); + headers.push( + request + .headers() + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); + let server_time = chrono::DateTime::::from(self.clock.now()).naive_utc() + + chrono::Duration::seconds(self.skew_seconds); + let (status, body) = if headers.len() == 1 { + ( + 403, + format!("{}Clock skew fixture", self.error_code), + ) + } else { + (200, String::new()) + }; + let response = http::Response::builder() + .status(status) + .header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string()) + .header("content-type", "application/xml") + .header("content-length", body.len()) + .body(SdkBody::from(body)) + .expect("clock skew fixture response"); + HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response"))) + } + } + + async fn clock_skew_client( + error_code: &'static str, + skew_seconds: i64, + retry: RemoteS3RetryPolicy, + ) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) { + let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new())); + let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000))); + let connector = SharedHttpConnector::new(ClockSkewConnector { + request_headers: Arc::clone(&headers), + error_code, + skew_seconds, + clock: clock.clone(), + }); + let mut spec = spec("s3.example.com", true); + spec.retry = retry; + let config = build_remote_s3_config(&spec) + .await + .expect("clock skew fixture uses the production outbound configuration") + .http_client(http_client_fn(move |_settings, _components| connector.clone())) + .time_source(clock.clone()) + .build(); + (S3Client::from_conf(config), headers, clock) + } + + #[tokio::test(start_paused = true)] + async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() { + for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] { + for skew_seconds in [-600, 600] { + let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await; + let initial = chrono::DateTime::::from(clock.now()).naive_utc(); + client + .get_object() + .bucket("bucket") + .key("object") + .send() + .await + .expect("clock skew GET must retry successfully"); + assert_eq!( + headers.lock().expect("captured requests").len(), + 2, + "{error_code}: GET needs exactly one retry" + ); + clock.0.fetch_add(17, Ordering::SeqCst); + // SDK signing time is independent of Tokio's retry/scheduler clock. + tokio::time::advance(Duration::from_secs(61)).await; + client + .head_bucket() + .bucket("bucket") + .send() + .await + .expect("subsequent HEAD must use the client's cached skew"); + let headers = headers.lock().expect("captured signed requests"); + assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt"); + assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock"); + assert_eq!( + signing_time(&headers[1]), + initial + chrono::Duration::seconds(skew_seconds), + "{error_code}: retry must apply the measured offset exactly" + ); + assert_eq!( + signing_time(&headers[2]), + initial + chrono::Duration::seconds(skew_seconds + 17), + "{error_code}: the next operation must apply cached skew to the advanced signing clock" + ); + let signature = |index: usize| { + recorded_header(&headers[index], "authorization") + .rsplit_once("Signature=") + .expect("SigV4 authorization contains a signature") + .1 + }; + assert_ne!( + signature(0), + signature(1), + "{error_code}: retry must be signed again after adjusting its date" + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn remote_s3_clock_skew_respects_one_attempt_policy() { + use aws_smithy_types::error::metadata::ProvideErrorMetadata; + + for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] { + for retry in [ + RemoteS3RetryPolicy::Disabled, + RemoteS3RetryPolicy::Standard { max_attempts: 1 }, + ] { + let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await; + let error = client + .get_object() + .bucket("bucket") + .key("object") + .send() + .await + .expect_err("clock skew must not override the caller's one-attempt budget"); + assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code)); + assert_eq!( + headers.lock().expect("captured requests").len(), + 1, + "{error_code}: {retry:?} must send exactly one request" + ); + } + } + } + #[test] fn path_style_auto_and_path_force_path_style() { assert!(PathStyle::Auto.force_path_style()); diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 4973f6831..94f3a6344 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -5493,6 +5493,7 @@ where fence.ensure_held()?; let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), ..Default::default() @@ -14412,6 +14413,7 @@ impl ECStore { encoded.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -14566,6 +14568,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(http_preconditions), ..Default::default() }, @@ -14957,6 +14960,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index e5eccba32..b98b294ca 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { dst_path: &str, external_guard: Option>, ) -> Result { + self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard) + .await + .result + } +} + +impl LocalDiskWrapper { + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + external_guard: Option>, + ) -> super::RenameDataObservation { let operation = self.clone(); let src_volume = src_volume.to_owned(); let src_path = src_path.to_owned(); @@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { } else { get_max_timeout_duration() }; - run_owned_mutation(external_guard, move || async move { - operation + let observed = run_owned_mutation(external_guard, move || async move { + let mut preflight_rejection = None; + let result = operation .track_disk_health_mutation( "rename_data", DiskMetricMutation::Write, || async { - operation - .disk - .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path) - .await + // Preserve the former DiskAPI future's single boxing boundary. + let observed = + Box::pin( + operation + .disk + .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path), + ) + .await; + preflight_rejection = observed.preflight_rejection; + observed.result }, timeout_duration, ) - .await + .await; + // Health tracking must observe the real disk error, not an Ok tuple. + Ok(super::RenameDataObservation { + result, + preflight_rejection, + }) }) - .await + .await; + observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error))) } } @@ -2588,6 +2617,46 @@ mod tests { assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1)); } + #[tokio::test] + async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() { + for source_exists in [false, true] { + for guarded in [false, true] { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")) + .expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + if source_exists { + disk.make_volume("source").await.expect("source volume should exist"); + } + let wrapper = LocalDiskWrapper::new(disk, false); + let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc); + let mut file_info = FileInfo::new("object", 1, 0); + file_info.mod_time = Some(::time::OffsetDateTime::now_utc()); + file_info.erasure.index = 1; + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard) + .await; + assert!(observed.rejected_before_publication(), "normal access rejection must carry proof"); + assert!(matches!(observed.result, Err(DiskError::VolumeNotFound))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok"); + assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded)); + + wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", None) + .await; + assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::FaultyDisk))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_writes, 0); + } + } + } + #[tokio::test] async fn local_disk_health_wrapper_counts_returned_availability_errors() { let dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 6baa92a3e..9cd683578 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(in crate::disk) use self::commit::LocalRenamePreflightRejection; +#[cfg(test)] +use self::commit::lock_rename_commit_directories; + +mod commit; + use crate::crash_inject::{self, CrashPoint}; use crate::data_usage::local_snapshot::ensure_data_usage_layout; use crate::diagnostics::get::{ @@ -26,10 +32,10 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, + RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, @@ -184,65 +190,6 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r } } -fn rollback_committed_rename_std( - dst_file_path: &Path, - new_data_path: Option<&Path>, - rollback_data_dir: Option, -) -> std::io::Result<()> { - if let Some(old_data_dir) = rollback_data_dir { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); - }; - let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); - std::fs::rename(backup_path, dst_file_path)?; - } else { - remove_file_if_exists(dst_file_path)?; - } - - if let Some(new_data_path) = new_data_path { - remove_dir_all_if_exists(new_data_path)?; - } - - Ok(()) -} - -fn rollback_inline_metadata_commit_std( - dst_file_path: &Path, - rollback_data_dir: Option, - local_rollback_path: Option<&Path>, -) -> std::io::Result<()> { - if let Some(backup_path) = local_rollback_path { - // The commit immediately before this rollback renamed the staged - // xl.meta from the same directory as `backup_path` onto - // `dst_file_path`, proving both paths are on the same filesystem. - // Unix rename atomically replaces the committed destination; never - // unlink it first or an interrupted rollback could lose xl.meta. - std::fs::rename(backup_path, dst_file_path)?; - } else { - rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; - } - Ok(()) -} - -fn create_local_inline_rollback_backup( - dst_file_path: &Path, - staging_file_path: &Path, - old_metadata: &[u8], -) -> std::io::Result { - let Some(staging_parent) = staging_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); - }; - let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); - remove_file_if_exists(&backup_path)?; - if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) - && let Err(err) = std::fs::write(&backup_path, old_metadata) - { - let _ = remove_file_if_exists(&backup_path); - return Err(err); - } - Ok(backup_path) -} - async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { let backup_dir = object_dir.join(rollback_dir.to_string()); fs::create_dir_all(&backup_dir).await.map_err(to_file_error)?; @@ -269,126 +216,6 @@ async fn restore_metadata_backup( Ok(()) } -async fn lock_rename_commit_directories( - source_parent: &Path, - destination_parent: &Path, - base_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result { - #[cfg(windows)] - let result = { - let source_parent = source_parent.to_path_buf(); - let destination_parent = destination_parent.to_path_buf(); - let base_dir = base_dir.to_path_buf(); - let publication_root = publication_root.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); - #[cfg(test)] - if result.is_ok() { - run_destination_commit_directory_preparation(&destination_parent); - } - result - }) - .await - }; - #[cfg(not(windows))] - let result = { - let _ = mutation_lease; - os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) - }; - - let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { - Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, - _ => err, - }); - - result.map_err(to_file_error).map_err(DiskError::from) -} - -async fn read_rename_destination_metadata( - file_path: &Path, - rename_commit_guard: &os::RenameCommitGuard, - mutation_lease: Arc, -) -> Result> { - #[cfg(windows)] - let result = { - let file_path = file_path.to_path_buf(); - let rename_commit_guard = rename_commit_guard.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) - }) - .await - }; - #[cfg(not(windows))] - let _ = (rename_commit_guard, mutation_lease); - #[cfg(not(windows))] - let result = match super::fs::read_file(file_path).await { - Ok(data) => Ok(Some(data)), - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - }; - - result - .map(|data| data.map(Bytes::from)) - .map_err(to_file_error) - .map_err(DiskError::from) -} - -async fn restore_renamed_data_source( - src_volume_dir: &Path, - src_data_path: &Path, - dst_data_path: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - if fs::symlink_metadata(src_data_path).await.is_ok() { - return Ok(()); - } - let result = - match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { - Ok(()) => Ok(()), - Err(DiskError::FileNotFound) => { - let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); - let destination_missing = matches!( - fs::symlink_metadata(dst_data_path).await, - Err(err) if err.kind() == ErrorKind::NotFound - ); - if source_exists && destination_missing { - Ok(()) - } else { - Err(DiskError::FileNotFound) - } - } - Err(err) => Err(err), - }; - if let Err(err) = &result { - warn!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "restore_staged_data_source_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Failed to restore staged data after a metadata commit was rejected" - ); - } - result -} - -async fn restore_published_data_source( - data_paths: Option<&(PathBuf, PathBuf)>, - src_volume_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - let Some((src_data_path, dst_data_path)) = data_paths else { - return Ok(()); - }; - restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await -} - async fn restore_delete_rollback( object_dir: &Path, xl_path: &Path, @@ -9040,7 +8867,6 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(level = "trace", skip_all)] async fn rename_data( &self, src_volume: &str, @@ -9049,966 +8875,8 @@ impl DiskAPI for LocalDisk { dst_volume: &str, dst_path: &str, ) -> Result { - crate::hp_guard!("LocalDisk::rename_data"); - let mut fi = fi; - // A non-force DeleteBucket must not remove a directory while a local - // object commit is publishing into it. The peer's empty scan remains - // optimistic; this lease establishes the local commit/delete order and - // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; - let quota_fence_token = - match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { - Some(value) => { - let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; - Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) - } - None if rustfs_utils::http::metadata_compat::contains_key_str( - &fi.metadata, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, - ) => - { - return Err(DiskError::FileCorrupt); - } - None => None, - }; - rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); - let quota_fence_claim = match quota_fence_token { - Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), - None => None, - }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; - if let Some(claim) = quota_fence_claim { - mutation_lease.attach_external_guard(claim); - } - if fi.is_legacy_indexed_delete_marker() { - fi.erasure.index = 0; - } - fi.validate_for_metadata_read()?; - // Snapshot the destination part paths before `fi` is consumed below. These - // are the descriptors a reader may hold for the version this call is about - // to replace (backlog#1145); readers build the identical string in - // `io_primitives`. An inline-data version has no parts and yields none. - let invalidate_part_paths: Vec = { - let data_dir = fi.data_dir.unwrap_or_default(); - fi.parts - .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) - .collect() - }; - let src_volume_dir = self.io_get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) - && let Err(e) = super::fs::access_std(&src_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?src_volume_dir, - operation = "rename_data_src_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) - && let Err(e) = super::fs::access_std(&dst_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?dst_volume_dir, - operation = "rename_data_dst_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - // xl.meta path - let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; - - // data_dir path - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir - .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = self.io_get_object_path( - src_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.io_get_object_path( - dst_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), - )?; - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - let no_inline = fi.data.is_none() && fi.size > 0; - // Captured before `fi` is consumed by add_version; gates the stale - // destination purge below. - let fi_healing = fi.is_healing(); - - // Resolved once for the whole commit so a concurrent configuration - // change can never leave a single rename_data half-synced. The tier is - // keyed on the destination volume: user data staged in scratch - // namespaces follows the configured tier, while commits into - // system-critical namespaces (IAM, config, bucket metadata) stay - // pinned to strict. - let durability = effective_durability(dst_volume); - - let src_file_parent = src_file_path - .parent() - .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; - let dst_file_parent = dst_file_path - .parent() - .ok_or_else(|| DiskError::other("missing object metadata parent"))?; - if !no_inline { - fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; - } - // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. - let rename_commit_guard = lock_rename_commit_directories( - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; - - if no_inline { - // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta - let mut xlmeta = FileMeta::new(); - // An existing dst xl.meta that fails to parse leaves `xlmeta` empty - // and gets overwritten by the commit below (pre-existing behavior); - // track that so the old-size observation reports unknown instead of - // a false `Absent` (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) - && let Ok(nmeta) = FileMeta::load(dst_buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() - && let Some(parent) = dst_file_path.parent() - { - skip_parent = parent.to_path_buf(); - } - - let version_id = fi.version_id.unwrap_or_default(); - let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = has_old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - if let Some(old_data_dir) = has_old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *old_data_dir); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_dst_buf = xlmeta.marshal_msg()?; - - // This tmp xl.meta is renamed onto dst_file_path at the commit - // point below, so only its contents must be durable before the - // rename (SyncMode::FileOnly); the dst parent directory is fsynced - // after the commit rename, and a crash before the rename means the - // PUT was never acknowledged. A metadata commit: relaxed tiers - // leave it to the page cache. - let tmp_meta_sync = if durability.syncs_commit_metadata() { - SyncMode::FileOnly - } else { - SyncMode::None - }; - // The tmp xl.meta write and the shard-file fdatasync are independent - // (disjoint paths) and both only need to be durable before the commit - // renames below, so run them concurrently to drop a blocking - // round-trip from the PUT commit critical path (rustfs/backlog#922 - // step 2). The "contents durable -> rename -> dst dir fsync" ordering - // is unchanged — both futures complete before any rename — which the - // rename_data crash-consistency harness (backlog#935) exercises. - // - // Shard durability: once rename_data succeeds the write is - // acknowledged, so data must not live only in the page cache. - // Multipart parts were already synced during rename_part, so their - // fdatasync here is a cheap no-op. A missing source dir is left for the - // rename below to report through the existing rollback path. Payload - // durability is kept by both strict and relaxed. - let tmp_meta_write = { - let src_file_path = src_file_path.clone(); - let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); - async move { - os::run_blocking_namespace_operation(mutation_lease, move || { - #[cfg(test)] - run_owned_file_write_before_open(&src_file_path); - let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( - &src_file_path, - &dst_file_path, - &rename_commit_guard, - )?; - prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; - Ok(prepared_metadata_source) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) - } - }; - let shard_sync = async { - if durability.syncs_data_shards() - && let Some((src_data_path, _)) = has_data_dir_path.as_ref() - && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await - && err.kind() != ErrorKind::NotFound - { - return Err::<(), DiskError>(to_file_error(err).into()); - } - Ok(()) - }; - let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); - // Surface a tmp-meta failure first (its prior serial position), then a - // shard-sync failure; either aborts before any rename, exactly as the - // sequential version did. - let prepared_metadata_source = tmp_meta_res?; - shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - std::fs::remove_file(&src_file_path).map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - // Heal reuses the version's data_dir, so for in-place corruption - // the destination dir still exists — and rename(2) cannot replace - // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge - // it first, healing commits only; fresh PUTs mint a new data_dir - // and never collide. Best effort: a real failure surfaces in the - // rename below. - if fi_healing - && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await - { - warn!( - event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - dst_path = ?dst_data_path, - error = ?err, - "Healing commit could not purge the stale destination data dir" - ); - } - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = os::rename_all_with_commit_guard( - src_data_path, - dst_data_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_data_path_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - #[cfg(test)] - if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - } - - // Crash-consistency injection: hard power loss after the data dir - // is in place but before xl.meta commits. No cleanup — the harness - // reopens the disk and asserts the object still reads as the old - // version (the staged data dir is a harmless orphan for GC). - if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { - return Err(DiskError::Unexpected); - } - - if should_fail_before_old_metadata_backup(dst_path) { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "test_fail_before_old_metadata_backup", - "Disk local rename flow failed before metadata commit" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::Unexpected); - } - - // The rollback backup stays where it is written (no rename) and is - // the sole restore source for a later undo_write, so under strict - // it keeps SyncMode::FileAndDir: contents and directory entry both - // durable. It is part of the metadata commit machinery, so relaxed - // tiers leave it to the page cache like the xl.meta it mirrors. - let backup_sync = if durability.syncs_commit_metadata() { - SyncMode::FileAndDir - } else { - SyncMode::None - }; - if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { - let backup_parent = dst_file_parent.join(old_data_dir.to_string()); - #[cfg(not(windows))] - if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { - Ok(guard) => guard, - Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::from(to_file_error(err))); - } - }; - let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); - if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { - #[cfg(windows)] - drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_bytes = dst_buf.clone(); - // Keep the volume, commit-tree, and exact destination-path - // guards in this task until the backup write and durability - // sync finish. A detached spawn_blocking writer could survive - // cancellation and later truncate a newer transaction's - // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { - #[cfg(test)] - run_owned_file_write_before_open(&backup_path); - backup_path_guard.write_file_for_path_access( - &backup_path, - backup_bytes.as_ref(), - backup_sync != SyncMode::None, - backup_sync == SyncMode::FileAndDir, - ) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from); - if let Err(err) = write_result { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "write_old_metadata_backup_failed", - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - } - - // Crash-consistency injection: hard power loss after the rollback - // backup is durable but before the xl.meta commit rename. No - // cleanup — the harness asserts the object still reads as the old - // version, since the destination xl.meta is untouched here. - if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - if let Err(err) = os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) + self.rename_data_inner(src_volume, src_path, fi, dst_volume, dst_path, &mut None) .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_metadata_failed", - src_path = ?src_file_path, - dst_path = ?dst_file_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); - if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - return Err(DiskError::Unexpected); - } - - // Crash-consistency injection: hard power loss immediately after the - // xl.meta commit rename but before the durability fsync. Unlike the - // graceful failpoint above, no rollback runs — the commit rename is - // already on disk, so the harness asserts the object reads back as - // the new version. - if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - // Persist the directory entries for both the data dir and xl.meta renames; - // without this the commit itself can vanish on power loss. Relaxed tiers - // accept that window (documented in docs/operations/durability-modes.md). - if durability.syncs_commit_metadata() - && let Some(parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // First PUT of an object creates its directory (and any missing prefix - // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The - // commit fsync above persists the object dir's *contents*, not its own - // entry in the bucket/prefix dir, so on power loss after ack the whole - // object dir could vanish (rustfs/backlog#922 step 4). For a new object - // (no prior xl.meta) fsync the ancestor chain from the object dir's - // parent up to and including the bucket so those new directory entries - // are durable. Overwrites already have a durable object dir. The - // starts_with guard bounds the walk to the bucket subtree. Relaxed/none - // accept the wider window, like the commit fsync above. - if has_dst_buf.is_none() && durability.syncs_commit_metadata() { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(dir) = ancestor { - if !dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // Same post-commit rollback window as above — drop cached - // dst part fds so readers re-open the restored inode - // (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if dir == dst_volume_dir.as_path() { - break; - } - ancestor = dir.parent(); - } - } - - // Publication and every rollback-capable durability step are now - // complete. Do not retain the Windows object identity guard while - // cleaning staging paths or invalidating cached descriptors. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = std::fs::remove_dir(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - rollback_data_dir, - cleanup_data_dir: has_old_data_dir, - sign: version_signature, - old_current_size, - }) - } else { - // Inline metadata preparation is blocking. The transaction lease is - // moved into that work so a timeout can release the async waiter without - // allowing a retry to reuse the deterministic staging path too early. - let src = src_file_path.clone(); - let dst = dst_file_path.clone(); - let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { - src_file_path.parent().map(|p| p.to_path_buf()) - } else { - None - }; - let dst_path_for_failpoint = dst_path.to_string(); - #[cfg(windows)] - let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); - let sync = durability.syncs_commit_metadata(); - #[cfg(test)] - run_inline_before_file_sync_admission(dst_path); - let mut file_sync_admission = if sync { - Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ) - } else { - None - }; - let prepare_inline_metadata = move || { - let mut prepared_metadata_source = - os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; - #[cfg(windows)] - let source_metadata_guard = - rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; - let mut xlmeta = FileMeta::new(); - // Same as the non-inline branch: an unparsable existing dst - // xl.meta must surface as unknown, not `Absent` - // (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(ref buf) = has_dst_buf { - if FileMeta::is_xl2_v1_format(buf) - && let Ok(nmeta) = FileMeta::load(buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let version_id = fi.version_id.unwrap_or_default(); - let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - let mut staged_rollback_path = None; - if let Some(d) = old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *d); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_buf = xlmeta.marshal_msg()?; - // Write the staged xl.meta. Inline objects carry their data inside - // xl.meta, so this is the durable preparation for the metadata commit: - // relaxed tiers do no per-object fsync here at all (aligned - // with MinIO's default), trading a documented power-loss - // window for latency. - prepared_metadata_source.write_all(&new_buf, sync)?; - run_inline_preparation_before_backup(&dst_path_for_failpoint); - if let Some(ref old_metadata) = has_dst_buf - && (rollback_data_dir.is_some() || sync || cfg!(test)) - { - #[cfg(windows)] - let backup_path = { - let backup_path = src - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? - .join(STORAGE_FORMAT_FILE_BACKUP); - source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; - backup_path - }; - #[cfg(not(windows))] - let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; - #[cfg(not(windows))] - if sync { - std::fs::File::open(&backup_path)?.sync_data()?; - } - staged_rollback_path = Some(backup_path); - } - - Ok::<_, std::io::Error>(( - rollback_data_dir, - old_data_dir, - version_signature, - old_current_size, - staged_rollback_path, - has_dst_buf.is_none(), - prepared_metadata_source, - )) - }; - let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await - } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await - } - .map_err(to_file_error) - .map_err(DiskError::from); - - let ( - rollback_data_dir, - cleanup_data_dir, - version_signature, - old_current_size, - mut local_rollback_path, - destination_was_absent, - prepared_metadata_source, - ) = match inline_preparation { - Ok(prepared) => prepared, - Err(err) => { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - }; - - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - remove_result.map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(DiskError::other("missing object metadata parent")); - }; - let backup_path = dst_parent - .join(rollback_data_dir.to_string()) - .join(STORAGE_FORMAT_FILE_BACKUP); - // rename_all acquires the backup path's namespace lease. Do not - // hold a disk admission while acquiring another namespace lock. - drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { - let _ = remove_file_if_exists(staged_backup); - return Err(err); - } - #[cfg(test)] - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - if sync { - file_sync_admission = Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ); - } - if let Some(admission) = file_sync_admission.as_ref() - && let Some(backup_parent) = backup_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - return Err(DiskError::from(to_file_error(err))); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - } - local_rollback_path = None; - } - - let commit_result = if should_fail_commit_rename(dst_path) { - Err(DiskError::other("test fail during metadata commit rename")) - } else { - os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - }; - if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - - let post_commit = async { - if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(std::io::Error::other("test fail after metadata commit")); - } - - // Persist the commit rename's directory entry across power loss. - if let Some(admission) = file_sync_admission.as_ref() - && let Some(dst_parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) - .await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // Same power-loss gap as the non-inline path (rustfs/backlog#922 - // step 4): a first PUT creates the object dir (and any missing - // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all - // never fsynced. The fsync above persists the object dir's contents, - // not its own entry, so for a new inline object fsync the ancestor - // chain up to and including the bucket. Overwrites already have a - // durable object dir; the starts_with guard bounds the walk. - if let Some(admission) = file_sync_admission.as_ref() - && destination_was_absent - { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(ancestor_dir) = ancestor { - if !ancestor_dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if ancestor_dir == dst_volume_dir.as_path() { - break; - } - ancestor = ancestor_dir.parent(); - } - } - - Ok::<(), std::io::Error>(()) - } - .await; - - // The disk admission protects the durability chain, not staging - // cleanup or cache invalidation after that chain has completed. - drop(file_sync_admission.take()); - - // A post-commit rollback (for example, a commit-metadata fsync - // failure under strict durability) restores the old metadata; drop any - // descriptors cached during the committed window before propagating the - // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so - // this is mostly defensive and keeps both commit branches consistent. - if let Err(err) = post_commit { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(DiskError::from(err)); - } - - // The commit no longer has a rollback path. Release the Windows - // object identity guard before best-effort staging cleanup. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - - // Cleanup - if let Some(ref cleanup) = cleanup_path { - let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; - } else if let Some(parent) = src_file_path.parent() { - let _ = std::fs::remove_dir(parent); - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: cleanup_data_dir, - rollback_data_dir, - cleanup_data_dir, - sign: version_signature, - old_current_size, - }) - } } #[tracing::instrument(level = "trace", skip_all)] @@ -11055,6 +9923,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf #[cfg(test)] mod test { + use super::commit::create_local_inline_rollback_backup; use super::*; use rustfs_filemeta::ErasureInfo; use std::io::{self, Write}; @@ -14102,6 +12971,196 @@ mod test { ); } + #[tokio::test] + async fn observed_rename_timeout_has_no_preflight_proof_and_retains_namespace_lease() { + use crate::disk::disk_store::LocalDiskWrapper; + use futures::FutureExt; + use std::sync::mpsc; + + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-timeout-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-timeout-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let staged_metadata = disk + .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}")) + .expect("staged metadata path should resolve"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_owned_file_write_before_open(&staged_metadata, move || { + entered_tx.send(()).expect("signal staged writer entry"); + // Dropping the sender also unblocks the syscall if the test fails. + let _ = release_rx.recv(); + }); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, None) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("staged writer waiter should run") + .expect("rename must enter the real staged metadata write"); + + // Advance only after the blocking syscall owns its lease and the wrapper's timer exists. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("wrapper timeout must not wait for the blocked syscall") + .expect("the wrapper waiter must not panic"); + assert!(!observed.rejected_before_publication(), "a timeout must carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::Timeout))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_errors_timeout, 1); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the blocked syscall must retain its volume guard" + ); + assert!( + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination) + .now_or_never() + .is_none(), + "a same-object mutation must still wait for the blocked syscall" + ); + assert_eq!(fs::read(&staged_part).await.expect("staged data must remain"), b"new-payload"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + + release_tx.send(()).expect("release timed-out staged writer"); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("the namespace lease must be released when the syscall drains"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("the volume guard must be released when the syscall drains"); + assert!( + !destination.join(STORAGE_FORMAT_FILE).exists(), + "timed-out waiter must not publish metadata later" + ); + }) + .await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn observed_rename_owned_task_panic_has_no_preflight_proof_and_releases_guard() { + use crate::disk::disk_store::LocalDiskWrapper; + use std::sync::mpsc; + + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-panic-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-panic-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let published_part = destination.join(data_dir.to_string()).join("part.1"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { + entered_tx.send(()).expect("signal data publication"); + let _ = release_rx.recv(); + // This hook runs in the owned async mutation, outside spawn_blocking. + panic!("injected observed rename owner panic after publication"); + }); + let external_guard = Arc::new(()); + let guard_probe = Arc::downgrade(&external_guard); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, Some(external_guard)) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("publication waiter should run") + .expect("rename must publish data before the injected owner panic"); + assert!(guard_probe.upgrade().is_some(), "the owned task must retain the publication guard"); + assert_eq!(fs::read(&published_part).await.expect("new data must be published"), b"new-payload"); + assert!(!staged_part.exists(), "the real data rename must have consumed staging"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the mutation must retain its volume guard" + ); + + release_tx.send(()).expect("release mutation owner into the injected panic"); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("owned task panic must reach the wrapper") + .expect("the wrapper must convert the inner task panic into an error"); + assert!( + !observed.rejected_before_publication(), + "a join failure must carry no local preflight proof" + ); + assert!(matches!(observed.result, Err(DiskError::Io(error)) if error.to_string() == "owned mutation task failed")); + assert!( + guard_probe.upgrade().is_none(), + "the guard must be released after the mutation owner unwinds" + ); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("panic must release the namespace lease"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("panic must release the volume guard"); + assert_eq!( + fs::read(&published_part).await.expect("published recovery data must remain"), + b"new-payload" + ); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn windows_and_unix_cancelled_staged_metadata_write_serializes_same_object_retry() { use std::sync::{Arc, mpsc}; diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs new file mode 100644 index 000000000..9d51bc1cd --- /dev/null +++ b/crates/ecstore/src/disk/local/commit.rs @@ -0,0 +1,1233 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Single-disk object rename publication and rollback. The shared execution core +//! retains instrumentation, mutation leases, and commit guards through the syscall. + +#[cfg(all(test, windows))] +use super::run_destination_commit_directory_preparation; +use super::{ + EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size, + remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, + run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, + should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit, + skip_access_checks, +}; +#[cfg(test)] +use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication}; +use crate::crash_inject::{self, CrashPoint}; +use crate::disk::{ + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, + error::{DiskError, Result}, + error_conv::{to_access_error, to_file_error}, + os, + os::{check_path_length, rename_all}, +}; +use bytes::Bytes; +use rustfs_filemeta::{FileInfo, FileMeta}; +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::fs; +use tracing::{info, warn}; +use uuid::Uuid; + +fn rollback_committed_rename_std( + dst_file_path: &Path, + new_data_path: Option<&Path>, + rollback_data_dir: Option, +) -> std::io::Result<()> { + if let Some(old_data_dir) = rollback_data_dir { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); + }; + let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + std::fs::rename(backup_path, dst_file_path)?; + } else { + remove_file_if_exists(dst_file_path)?; + } + + if let Some(new_data_path) = new_data_path { + remove_dir_all_if_exists(new_data_path)?; + } + + Ok(()) +} + +fn rollback_inline_metadata_commit_std( + dst_file_path: &Path, + rollback_data_dir: Option, + local_rollback_path: Option<&Path>, +) -> std::io::Result<()> { + if let Some(backup_path) = local_rollback_path { + // The commit immediately before this rollback renamed the staged + // xl.meta from the same directory as `backup_path` onto + // `dst_file_path`, proving both paths are on the same filesystem. + // Unix rename atomically replaces the committed destination; never + // unlink it first or an interrupted rollback could lose xl.meta. + std::fs::rename(backup_path, dst_file_path)?; + } else { + rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; + } + Ok(()) +} + +pub(super) fn create_local_inline_rollback_backup( + dst_file_path: &Path, + staging_file_path: &Path, + old_metadata: &[u8], +) -> std::io::Result { + let Some(staging_parent) = staging_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); + }; + let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); + remove_file_if_exists(&backup_path)?; + if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) + && let Err(err) = std::fs::write(&backup_path, old_metadata) + { + let _ = remove_file_if_exists(&backup_path); + return Err(err); + } + Ok(backup_path) +} + +pub(super) async fn lock_rename_commit_directories( + source_parent: &Path, + destination_parent: &Path, + base_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result { + #[cfg(windows)] + let result = { + let source_parent = source_parent.to_path_buf(); + let destination_parent = destination_parent.to_path_buf(); + let base_dir = base_dir.to_path_buf(); + let publication_root = publication_root.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); + #[cfg(test)] + if result.is_ok() { + run_destination_commit_directory_preparation(&destination_parent); + } + result + }) + .await + }; + #[cfg(not(windows))] + let result = { + let _ = mutation_lease; + os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) + }; + + let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { + Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, + _ => err, + }); + + result.map_err(to_file_error).map_err(DiskError::from) +} + +async fn read_rename_destination_metadata( + file_path: &Path, + rename_commit_guard: &os::RenameCommitGuard, + mutation_lease: Arc, +) -> Result> { + #[cfg(windows)] + let result = { + let file_path = file_path.to_path_buf(); + let rename_commit_guard = rename_commit_guard.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) + }) + .await + }; + #[cfg(not(windows))] + let _ = (rename_commit_guard, mutation_lease); + #[cfg(not(windows))] + let result = match super::super::fs::read_file(file_path).await { + Ok(data) => Ok(Some(data)), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + + result + .map(|data| data.map(Bytes::from)) + .map_err(to_file_error) + .map_err(DiskError::from) +} + +async fn restore_renamed_data_source( + src_volume_dir: &Path, + src_data_path: &Path, + dst_data_path: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + if fs::symlink_metadata(src_data_path).await.is_ok() { + return Ok(()); + } + let result = + match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { + Ok(()) => Ok(()), + Err(DiskError::FileNotFound) => { + let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); + let destination_missing = matches!( + fs::symlink_metadata(dst_data_path).await, + Err(err) if err.kind() == ErrorKind::NotFound + ); + if source_exists && destination_missing { + Ok(()) + } else { + Err(DiskError::FileNotFound) + } + } + Err(err) => Err(err), + }; + if let Err(err) = &result { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "restore_staged_data_source_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Failed to restore staged data after a metadata commit was rejected" + ); + } + result +} + +async fn restore_published_data_source( + data_paths: Option<&(PathBuf, PathBuf)>, + src_volume_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + let Some((src_data_path, dst_data_path)) = data_paths else { + return Ok(()); + }; + restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await +} + +/// Proof produced only when the local rename returns at an existing access +/// preflight, before metadata, backups, or object data can be published. +#[derive(Debug)] +pub(in crate::disk) struct LocalRenamePreflightRejection(()); + +impl LocalDisk { + #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)] + pub(super) async fn rename_data_inner( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + preflight_rejection: &mut Option, + ) -> Result { + crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::super::fs::access_std(&src_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::super::fs::access_std(&dst_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } + + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> super::super::RenameDataObservation { + let mut preflight_rejection = None; + let result = self + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .await; + super::super::RenameDataObservation { + result, + preflight_rejection, + } + } +} diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 7801274ef..c2f2c52b4 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -75,6 +75,25 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +/// Local preflight evidence stays outside DiskAPI and the RPC response format. +pub(crate) struct RenameDataObservation { + pub(crate) result: Result, + preflight_rejection: Option, +} + +impl RenameDataObservation { + fn unknown(result: Result) -> Self { + Self { + result, + preflight_rejection: None, + } + } + + pub(crate) fn rejected_before_publication(&self) -> bool { + self.result.is_err() && self.preflight_rejection.is_some() + } +} + const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; @@ -711,6 +730,36 @@ impl Disk { .await } + pub(crate) async fn rename_data_borrowed_with_fence_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + scanner_publication_lease_token: Option, + ) -> RenameDataObservation { + match self { + Disk::Local(local_disk) => { + local_disk + .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None) + .await + } + Disk::Remote(remote_disk) => RenameDataObservation::unknown( + remote_disk + .rename_data_borrowed_with_fence( + src_volume, + src_path, + fi, + dst_volume, + dst_path, + scanner_publication_lease_token, + ) + .await, + ), + } + } + pub(crate) async fn rename_data_borrowed_with_fence( &self, src_volume: &str, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 85e270679..701596cbf 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink { } } +/// Internal PUT completion boundary; this does not change fsync or write quorum. +#[doc(hidden)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum WriteCompletion { + /// Return at write quorum when the commit owner can retain its guards. + #[default] + Quorum, + /// Drain the rename fan-out before returning. Minority failures still heal + /// after a successful quorum commit; this does not require every disk to succeed. + TailDrained, +} + #[derive(Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -896,6 +908,10 @@ pub struct ObjectOptions { /// Persisted bucket incarnation observed before authorization. pub expected_bucket_incarnation_id: Option, pub no_lock: bool, + /// Control-plane writers that immediately read or CAS the same namespace + /// key use TailDrained without changing namespace lock ownership. + #[doc(hidden)] + pub write_completion: WriteCompletion, /// True when an upper layer already holds the object read lock before /// forwarding a no_lock read to the set layer. pub metadata_cache_safe: bool, diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index f2c4ddeb6..59b4d6f4e 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend { Ok(version) } - async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result { + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { self.precondition().await?; let barrier = self.inner.get_barrier.lock().await.take(); if let Some(barrier) = barrier { @@ -719,6 +719,9 @@ impl WarmBackend for MockWarmBackend { let Some(stored) = objects.get(object) else { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found")); }; + if !rv.is_empty() && stored.remote_version_id != rv { + return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion")); + } let bytes = &stored.bytes; let start = opts.start_offset.max(0) as usize; diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 887015a1e..af24423fa 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -2346,6 +2346,10 @@ impl WarmBackend for SharedWarmBackendProxy { self.0.probe_transition_candidate(object).await } + async fn probe_transition_version(&self, object: &str, remote_version_id: &str) -> io::Result { + self.0.probe_transition_version(object, remote_version_id).await + } + async fn in_use(&self) -> io::Result { self.0.in_use().await } @@ -2458,6 +2462,15 @@ impl TierOperationLease { Ok(()) } + pub(crate) async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> io::Result { + self.validate_remote_version_id(remote_version_id)?; + self.inner.driver.probe_transition_version(object, remote_version_id).await + } + pub(crate) fn is_current_generation(&self) -> bool { lock_unpoisoned(&self.runtime) .generations diff --git a/crates/ecstore/src/services/tier/tier_admin.rs b/crates/ecstore/src/services/tier/tier_admin.rs index 32cd844a0..6c3bc1c4a 100644 --- a/crates/ecstore/src/services/tier/tier_admin.rs +++ b/crates/ecstore/src/services/tier/tier_admin.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -145,7 +143,7 @@ mod tests { assert_eq!(creds.access_key, "access"); assert_eq!(creds.secret_key, "secret"); - assert_eq!(creds.creds_json.as_slice(), &service_account[..]); + assert_eq!(creds.creds_json.as_slice(), service_account); let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode"); assert_eq!(wire["access"], "access"); @@ -162,7 +160,7 @@ mod tests { .expect("the former RustFS field names and byte-array encoding should remain readable"); assert_eq!(legacy.access_key, "legacy-access"); assert_eq!(legacy.secret_key, "legacy-secret"); - assert_eq!(legacy.creds_json.as_slice(), &service_account[..]); + assert_eq!(legacy.creds_json.as_slice(), service_account); } #[test] diff --git a/crates/ecstore/src/services/tier/tier_mutation_intent.rs b/crates/ecstore/src/services/tier/tier_mutation_intent.rs index 288b02969..eef2dcf7a 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_intent.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_intent.rs @@ -460,6 +460,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -556,6 +557,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/services/tier/tier_probe_intent.rs b/crates/ecstore/src/services/tier/tier_probe_intent.rs index 3d11402a5..b3d96dc05 100644 --- a/crates/ecstore/src/services/tier/tier_probe_intent.rs +++ b/crates/ecstore/src/services/tier/tier_probe_intent.rs @@ -494,6 +494,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -549,6 +550,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current.record_etag.clone()), ..Default::default() diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index ee48c116c..b5cf4ab38 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -40,6 +40,7 @@ use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value}; use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}; use rustfs_s3_client::{ admin_handler_utils::AdminError, + api_error_response::to_error_response, api_put_object::{AdvancedPutOptions, PutObjectOptions}, transition_api::{ReadCloser, ReaderImpl}, }; @@ -48,11 +49,14 @@ use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; -use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}; use s3s::header::{ X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS, X_AMZ_STORAGE_CLASS, }; +use s3s::{ + S3ErrorCode, + dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}, +}; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -141,6 +145,42 @@ pub trait WarmBackend { async fn probe_transition_candidate(&self, _object: &str) -> Result { Ok(TransitionCandidateProbe::Unsupported) } + async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> Result { + if remote_version_id.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "an exact tier probe requires a remote version ID", + )); + } + self.validate_remote_version_id(remote_version_id)?; + match self + .get( + object, + remote_version_id, + WarmBackendGetOpts { + start_offset: 0, + length: 1, + }, + ) + .await + { + Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())), + Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => { + Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())) + } + Err(err) + if err.kind() == std::io::ErrorKind::NotFound + || matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) => + { + Ok(TransitionCandidateProbe::Missing) + } + Err(err) => Err(err), + } + } async fn in_use(&self) -> Result; } @@ -437,6 +477,17 @@ impl WarmBackend for MeteredWarmBackend { Self::record(TierRequestOperation::Probe, result) } + async fn probe_transition_version( + &self, + object: &str, + remote_version_id: &str, + ) -> Result { + Self::record( + TierRequestOperation::Probe, + self.inner.probe_transition_version(object, remote_version_id).await, + ) + } + async fn in_use(&self) -> Result { Self::record(TierRequestOperation::InUse, self.inner.in_use().await) } diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index 27fb7decd..f08c8fe2e 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index ed9ef0cf8..073501e65 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 03c2df988..1aa6b538c 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::{HashMap, HashSet}; use std::future::Future; @@ -146,11 +144,11 @@ pub struct WarmBackendGCS { impl WarmBackendGCS { pub async fn new(conf: &TierGCS, tier: &str) -> Result { - if conf.creds == "" { + if conf.creds.is_empty() { return Err(std::io::Error::other("both access and secret keys are required")); } - if conf.bucket == "" { + if conf.bucket.is_empty() { return Err(std::io::Error::other("no bucket name was provided")); } @@ -195,11 +193,11 @@ impl WarmBackendGCS { } pub fn get_dest(&self, object: &str) -> String { - let mut dest_obj = object.to_string(); - if self.prefix != "" { - dest_obj = format!("{}/{}", &self.prefix, object); + if self.prefix.is_empty() { + object.to_string() + } else { + format!("{}/{}", self.prefix, object) } - return dest_obj; } } @@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS { let bucket = gcs_bucket_resource_name(&self.bucket); let Ok(res) = Box::pin( self.client - .write_object(&bucket, &self.get_dest(object), Bytes::from(d)) + .write_object(&bucket, self.get_dest(object), Bytes::from(d)) .send_buffered(), ) .await @@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS { async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { let bucket = gcs_bucket_resource_name(&self.bucket); - let mut req = self.client.read_object(&bucket, &self.get_dest(object)); + let mut req = self.client.read_object(&bucket, self.get_dest(object)); let mut max_response_bytes = None; if let Some(generation) = parse_generation(rv)? { req = req.set_generation(generation); diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index 73ced2cef..e626f6c9f 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 020c2e319..6baef149b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs index 071ccefad..79a47d600 100644 --- a/crates/ecstore/src/services/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 32821c427..0bb19bcdf 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 5462fc52c..b830ea7f2 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -529,6 +529,10 @@ mod tests { "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\nInvalidRangeempty version", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\nNoSuchVersionmissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", ]; let mut requests = Vec::new(); for response in responses { @@ -622,15 +626,52 @@ mod tests { .await .expect_err("an authorization failure must not be mistaken for a missing key"); assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied); + assert_eq!( + backend + .probe_transition_candidate("delete-marker-hidden") + .await + .expect("a current delete marker should hide the data version"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_version("delete-marker-hidden", "historical-version") + .await + .expect("the stored historical version should be probed exactly"), + TransitionCandidateProbe::VersionedPresent("historical-version".to_string()) + ); + assert_eq!( + backend + .probe_transition_version("delete-marker-hidden", "missing-version") + .await + .expect("a missing exact version should be classified"), + TransitionCandidateProbe::Missing + ); + assert_eq!( + backend + .probe_transition_version("missing-object", "historical-version") + .await + .expect("a missing key for an exact version probe should be classified"), + TransitionCandidateProbe::Missing + ); let requests = fixture.await.expect("candidate fixture should join"); - for request in requests { + for request in &requests[..6] { let request = request.to_ascii_lowercase(); assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET"); assert!(request.contains("\r\nrange: bytes=0-0\r\n")); assert!(!request.contains("?versioning")); assert!(!request.contains("?versions")); } + for request in &requests[6..] { + let request = request.to_ascii_lowercase(); + assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET"); + assert!(request.contains("\r\nrange: bytes=0-0\r\n")); + } + assert!(!requests[5].to_ascii_lowercase().contains("versionid=")); + assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version")); + assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version")); + assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version")); } fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index 20eb7ee81..8045b3332 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -15,8 +15,6 @@ #![allow(unused_variables)] #![allow(unused_mut)] #![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use std::collections::HashMap; diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index c07a013de..a7c52549d 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1492,6 +1492,7 @@ pub(in crate::set_disk) fn record_read_repair_dedup(reason: &'static str) { counter!("rustfs_heal_read_repair_dedup_total", "reason" => reason).increment(1); } +#[derive(Debug)] pub(in crate::set_disk) enum ReadRepairAdmissionOutcome { Response(HealAdmissionResult), Failed(String), @@ -3838,6 +3839,286 @@ pub(in crate::set_disk) struct RenameTailOutcome { pub(in crate::set_disk) cleanup: Vec, } +const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; + +#[derive(Clone, Copy)] +enum RenameDispatchState { + NotDispatched, + RejectedBeforePublication, + MayHavePublished, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RenameRollbackOutcome { + NotAttempted(DiskError), + RejectedBeforePublication(DiskError), + Indeterminate(DiskError), + Succeeded, + Failed(DiskError), + Panicked, + Cancelled, +} + +impl RenameRollbackOutcome { + fn stage(&self) -> &'static str { + match self { + Self::NotAttempted(_) => "rename_not_dispatched", + Self::RejectedBeforePublication(_) => "rename_rejected_before_publication", + Self::Indeterminate(_) => "rename_indeterminate", + Self::Succeeded => "undo_succeeded", + Self::Failed(_) => "undo_failed", + Self::Panicked => "undo_panicked", + Self::Cancelled => "undo_cancelled", + } + } + + fn undo_attempted(&self) -> bool { + matches!(self, Self::Succeeded | Self::Failed(_) | Self::Panicked | Self::Cancelled) + } + + fn needs_recovery(&self) -> bool { + matches!(self, Self::Indeterminate(_) | Self::Failed(_) | Self::Panicked | Self::Cancelled) + } +} + +#[derive(Debug, Clone)] +struct RenameRollbackDiskOutcome { + disk_index: usize, + rollback_dir: Option, + outcome: RenameRollbackOutcome, +} + +#[derive(Debug)] +struct RenameRollbackReport { + disks: Vec, +} + +/// Shares rollback completion with the staging owner without replacing the +/// original disk/quorum error returned by the rename operation. +#[derive(Clone, Default)] +pub(in crate::set_disk) struct RenameRollbackReceipt(Arc>); + +impl RenameRollbackReceipt { + pub(in crate::set_disk) fn is_incomplete(&self) -> bool { + self.0 + .get() + .is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.needs_recovery())) + } +} + +async fn inspect_incomplete_rename_rollback( + disks: &[Option], + bucket: &str, + object: &str, + submitter: ReadRepairAdmissionSubmitter, +) -> ReadRepairAdmissionOutcome { + let location = disks.iter().flatten().next().map(|disk| disk.get_disk_location()); + let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(HealChannelPriority::High), + location.as_ref().and_then(|location| location.pool_idx), + location.as_ref().and_then(|location| location.set_idx), + ); + // A failed write's surviving minority is not an authoritative heal source. + // Request inspection only: MRF PartialWrite would schedule mutating repair. + request.dry_run = Some(true); + request.remove_corrupted = Some(false); + request.recreate_missing = Some(false); + request.update_parity = Some(false); + request.recursive = Some(false); + match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await { + Ok(result) => result, + Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()), + } +} + +fn rename_rollback_task_outcome( + result: std::result::Result, tokio::task::JoinError>, +) -> RenameRollbackOutcome { + match result { + Ok(Ok(())) => RenameRollbackOutcome::Succeeded, + Ok(Err(err)) => RenameRollbackOutcome::Failed(err), + Err(err) if err.is_panic() => RenameRollbackOutcome::Panicked, + Err(_) => RenameRollbackOutcome::Cancelled, + } +} + +async fn rollback_failed_rename( + disks: &[Option], + file_infos: Vec, + errs: &[Option], + dispatch_states: &[RenameDispatchState], + rollback_dirs: &[Option], + dst: (&str, &str), + receipt: Option, +) { + let owned_disks = disks.to_vec(); + let owned_errs = errs.to_vec(); + let owned_dispatch_states = dispatch_states.to_vec(); + let owned_dirs = rollback_dirs.to_vec(); + let owned_dst = (dst.0.to_string(), dst.1.to_string()); + let coordinator_failure_receipt = receipt.clone(); + // Own both undo mutations and their accounting: a cancelled requester must + // not leave detached disk tasks without the recovery evidence they produce. + let rollback = tokio::spawn(async move { + let disks = owned_disks.as_slice(); + let errs = owned_errs.as_slice(); + let dispatch_states = owned_dispatch_states.as_slice(); + let rollback_dirs = owned_dirs.as_slice(); + let dst = (owned_dst.0.as_str(), owned_dst.1.as_str()); + let mut file_infos = file_infos; + + let (bucket, object) = dst; + let mut outcomes = Vec::with_capacity(disks.len()); + let mut tasks = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let rollback_dir = rollback_dirs[disk_index]; + let outcome = match &errs[disk_index] { + Some(err) => match dispatch_states[disk_index] { + RenameDispatchState::NotDispatched => RenameRollbackOutcome::NotAttempted(err.clone()), + RenameDispatchState::RejectedBeforePublication => { + RenameRollbackOutcome::RejectedBeforePublication(err.clone()) + } + RenameDispatchState::MayHavePublished => RenameRollbackOutcome::Indeterminate(err.clone()), + }, + None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), + }; + outcomes.push(RenameRollbackDiskOutcome { + disk_index, + rollback_dir, + outcome, + }); + if errs[disk_index].is_some() { + continue; + } + let Some(disk) = disk.clone() else { + continue; + }; + let fi = std::mem::take(&mut file_infos[disk_index]); + let bucket = bucket.to_string(); + let object = object.to_string(); + let task = tokio::spawn(async move { + #[allow(clippy::let_unit_value)] + let _task_guard = SetDisks::rename_fanout_task_guard(&object); + SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; + #[cfg(test)] + rollback_fault_injection::before_undo(&object, disk_index)?; + disk.delete_version( + &bucket, + &object, + fi, + false, + DeleteOptions { + undo_write: true, + old_data_dir: rollback_dir, + ..Default::default() + }, + ) + .await + }); + tasks.push(async move { (disk_index, task.await) }); + } + for (disk_index, result) in join_all(tasks).await { + outcomes[disk_index].outcome = rename_rollback_task_outcome(result); + } + + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; + }); + if rollback.await.is_err() { + record_indeterminate_rename(disks, dst, coordinator_failure_receipt).await; + } +} + +async fn record_indeterminate_rename(disks: &[Option], dst: (&str, &str), receipt: Option) { + let outcomes = disks + .iter() + .enumerate() + .map(|(disk_index, disk)| RenameRollbackDiskOutcome { + disk_index, + rollback_dir: None, + outcome: if disk.is_some() { + RenameRollbackOutcome::Indeterminate(DiskError::Unexpected) + } else { + RenameRollbackOutcome::NotAttempted(DiskError::DiskNotFound) + }, + }) + .collect(); + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; +} + +async fn record_rename_rollback_outcomes( + disks: &[Option], + outcomes: Vec, + dst: (&str, &str), + receipt: Option, +) { + let (bucket, object) = dst; + let attempted = outcomes.iter().filter(|disk| disk.outcome.undo_attempted()).count(); + let failed = outcomes + .iter() + .filter(|disk| disk.outcome.undo_attempted() && disk.outcome.needs_recovery()) + .count(); + let indeterminate = outcomes + .iter() + .filter(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(_))) + .count(); + let succeeded = attempted - failed; + for disk in &outcomes { + counter!("rustfs_rename_rollback_disks_total", "stage" => disk.outcome.stage()).increment(1); + if disk.outcome.needs_recovery() { + let location = disks[disk.disk_index].as_ref().map(|disk| disk.get_disk_location()); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = disk.outcome.stage(), + pool_index = ?location.as_ref().and_then(|location| location.pool_idx), + set_index = ?location.as_ref().and_then(|location| location.set_idx), + disk_index = disk.disk_index, + bucket, + object, + rollback_dir = ?disk.rollback_dir, + outcome = ?disk.outcome, + attempted, + succeeded, + failed, + indeterminate, + "rename rollback incomplete; preserve recovery material" + ); + } + } + if let Some(receipt) = receipt { + let _ = receipt.0.set(RenameRollbackReport { disks: outcomes }); + } + if failed > 0 || indeterminate > 0 { + let result = inspect_incomplete_rename_rollback(disks, bucket, object, send_read_repair_heal_request).await; + let admission = match &result { + ReadRepairAdmissionOutcome::Response(response) => response.result_label(), + ReadRepairAdmissionOutcome::Failed(_) => "failed", + }; + counter!("rustfs_rename_rollback_inspection_total", "admission" => admission).increment(1); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = "inspection_admission", + bucket, + object, + attempted, + succeeded, + failed, + indeterminate, + admission, + outcome = ?result, + "rename rollback inspection requested; recovery remains incomplete" + ); + } +} + /// Options shared by the normal and early-ack rename fanouts. Keeping the /// quorum and optional scanner lease map together avoids widening either /// fanout helper's argument list while preserving the fence semantics. @@ -3845,6 +4126,7 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> { write_quorum: usize, scanner_publication_lease_tokens: Option<&'a HashMap>, scanner_publication_commit_scope: Option, + rollback_receipt: Option, } impl<'a> RenameDataFenceOptions<'a> { @@ -3856,9 +4138,15 @@ impl<'a> RenameDataFenceOptions<'a> { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: None, + rollback_receipt: None, } } + pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self { + self.rollback_receipt = Some(receipt); + self + } + pub(in crate::set_disk) fn with_publication_scope( mut self, scanner_publication_commit_scope: Option, @@ -4224,6 +4512,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: _scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4247,6 +4536,7 @@ impl SetDisks { let dst_object = Arc::new(dst_object.to_string()); let (commit_tx, commit_rx) = tokio::sync::oneshot::channel(); + let coordinator_failure_receipt = rollback_receipt.clone(); let tail_drain = tokio::spawn({ let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); @@ -4269,7 +4559,8 @@ impl SetDisks { let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); tasks.spawn(async move { - let result = std::panic::AssertUnwindSafe(async move { + let mut dispatch_state = RenameDispatchState::NotDispatched; + let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); @@ -4294,8 +4585,9 @@ impl SetDisks { } let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - let result = disk - .rename_data_borrowed_with_fence( + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( &src_bucket, &src_object, &file_info, @@ -4304,6 +4596,12 @@ impl SetDisks { scanner_publication_lease_token, ) .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } if let Some(disk_wait_started) = disk_wait_started { let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; rustfs_io_metrics::record_put_object_stage_duration( @@ -4325,11 +4623,14 @@ impl SetDisks { }; rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } result }) .catch_unwind() .await; - (i, result) + (i, dispatch_state, result) }); } @@ -4339,6 +4640,8 @@ impl SetDisks { let mut fanout_panic = 0usize; let mut results_seen = 0usize; let mut errs = vec![Some(DiskError::DiskNotFound); disk_count]; + // Missing task results cannot prove that a disk mutation never ran. + let mut dispatch_states = vec![RenameDispatchState::MayHavePublished; disk_count]; let mut disk_versions = vec![None; disk_count]; let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; @@ -4349,7 +4652,8 @@ impl SetDisks { while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { - Ok((idx, Ok(Ok(res)))) => { + Ok((idx, dispatch_state, Ok(Ok(res)))) => { + dispatch_states[idx] = dispatch_state; data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; @@ -4357,10 +4661,12 @@ impl SetDisks { errs[idx] = None; success_count += 1; } - Ok((idx, Ok(Err(err)))) => { + Ok((idx, dispatch_state, Ok(Err(err)))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(err); } - Ok((idx, Err(_))) => { + Ok((idx, dispatch_state, Err(_))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(DiskError::Unexpected); fanout_panic += 1; } @@ -4390,6 +4696,8 @@ impl SetDisks { } } + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); if rustfs_io_metrics::put_stage_metrics_enabled() { let fanout_success = errs.iter().filter(|err| err.is_none()).count(); let fanout_error = errs.len().saturating_sub(fanout_success + fanout_panic); @@ -4405,36 +4713,16 @@ impl SetDisks { if !sent_commit { let ret_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum).unwrap_or(DiskError::Unexpected); - let mut rollbacks = Vec::new(); - let mut rollback_file_infos = file_infos; - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = coordinator_disks[i].as_ref() { - let fi = std::mem::take(&mut rollback_file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = fanout_dst_bucket.clone(); - let dst_object = fanout_dst_object.clone(); - rollbacks.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - let _ = join_all(rollbacks).await; + rollback_failed_rename( + &coordinator_disks, + file_infos, + &errs, + &dispatch_states, + &data_dirs, + (&fanout_dst_bucket, &fanout_dst_object), + rollback_receipt, + ) + .await; if let Some(commit_tx) = commit_tx.take() { let _ = commit_tx.send(Err(ret_err)); } @@ -4524,7 +4812,13 @@ impl SetDisks { }); let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); - let commit = commit_rx.await.map_err(|_| DiskError::Unexpected)?; + let commit = match commit_rx.await { + Ok(commit) => commit, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), coordinator_failure_receipt).await; + return Err(DiskError::Unexpected); + } + }; rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, @@ -4582,6 +4876,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4637,81 +4932,100 @@ impl SetDisks { let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let publication_scope = scanner_publication_commit_scope.clone(); - std::panic::AssertUnwindSafe(async move { - // Test-only introspection guard: counts this operation as - // in-flight for the whole body. Compiles to `()` in production. - #[allow(clippy::let_unit_value)] - let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); + async move { + let mut dispatch_state = RenameDispatchState::NotDispatched; + let result = std::panic::AssertUnwindSafe(async { + // Test-only introspection guard: counts this operation as + // in-flight for the whole body. Compiles to `()` in production. + #[allow(clippy::let_unit_value)] + let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); - let Some(disk) = disk else { - return Err(DiskError::DiskNotFound); - }; - - let is_delete_marker = file_info.is_canonical_delete_marker(); - let mut local_file_info; - let file_info = if file_info.erasure.index == 0 { - local_file_info = file_info.clone(); - local_file_info.erasure.index = i + 1; - &local_file_info - } else { - file_info - }; - if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { - return Err(DiskError::FileCorrupt); - } - - // Test-only awaitable pause point right before the disk rename. - // A no-op immediately-ready future in production. - Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - - if let Some(err) = Self::rename_injected_error(&dst_object, i) { - return Err(err); - } - - if let Some(scope) = publication_scope.as_ref() - && !scope.can_commit() - { - let _ = scope.mark_indeterminate(); - return Err(DiskError::other("scanner publication commit scope deadline or cancellation reached")); - } - - let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - let result = disk - .rename_data_borrowed_with_fence( - &src_bucket, - &src_object, - file_info, - &dst_bucket, - &dst_object, - scanner_publication_lease_token, - ) - .await; - if let Some(disk_wait_started) = disk_wait_started { - let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; - rustfs_io_metrics::record_put_object_stage_duration( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, - duration_ms, - ); - let position = if result.is_ok() { - let rank = successful_rename_completion_rank - .as_ref() - .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) - .unwrap_or(1); - if rank <= write_quorum { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL - } - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + let Some(disk) = disk else { + return Err(DiskError::DiskNotFound); }; - rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); - } - result - }) - .catch_unwind() + + let is_delete_marker = file_info.is_canonical_delete_marker(); + let mut local_file_info; + let file_info = if file_info.erasure.index == 0 { + local_file_info = file_info.clone(); + local_file_info.erasure.index = i + 1; + &local_file_info + } else { + file_info + }; + if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { + return Err(DiskError::FileCorrupt); + } + + // Test-only awaitable pause point right before the disk rename. + // A no-op immediately-ready future in production. + Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; + + if let Some(err) = Self::rename_injected_error(&dst_object, i) { + return Err(err); + } + + if let Some(scope) = publication_scope.as_ref() + && !scope.can_commit() + { + let _ = scope.mark_indeterminate(); + return Err(DiskError::other( + "scanner publication commit scope deadline or cancellation reached", + )); + } + + let disk_wait_started = rustfs_io_metrics::put_stage_timer(); + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( + &src_bucket, + &src_object, + file_info, + &dst_bucket, + &dst_object, + scanner_publication_lease_token, + ) + .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } + if let Some(disk_wait_started) = disk_wait_started { + let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + duration_ms, + ); + let position = if result.is_ok() { + let rank = successful_rename_completion_rank + .as_ref() + .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) + .unwrap_or(1); + if rank <= write_quorum { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL + } + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + }; + rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); + } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } + result + }) + .catch_unwind() + .await; + (dispatch_state, result) + } }); let results = join_all(futures).await; + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); (results, fanout_file_infos) }); @@ -4726,12 +5040,18 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, ); - let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; + let (results, mut file_infos) = match fanout_result { + Ok(result) => result, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), rollback_receipt).await; + return Err(DiskError::Unexpected); + } + }; if rustfs_io_metrics::put_stage_metrics_enabled() { let mut fanout_success = 0; let mut fanout_error = 0; let mut fanout_panic = 0; - for result in &results { + for (_, result) in &results { match result { Ok(Ok(_)) => fanout_success += 1, Ok(Err(_)) => fanout_error += 1, @@ -4747,7 +5067,9 @@ impl SetDisks { ); } - for (idx, result) in results.iter().enumerate() { + let mut dispatch_states = Vec::with_capacity(results.len()); + for (idx, (dispatch_state, result)) in results.iter().enumerate() { + dispatch_states.push(*dispatch_state); match result { Ok(Ok(res)) => { data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); @@ -4793,36 +5115,7 @@ impl SetDisks { ); } - let mut futures = Vec::with_capacity(disks.len()); if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = disks[i].as_ref() { - let fi = std::mem::take(&mut file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = dst_bucket.clone(); - let dst_object = dst_object.clone(); - futures.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - if issue3031_diag_enabled() { warn!( target: "rustfs_ecstore::set_disk", @@ -4838,23 +5131,16 @@ impl SetDisks { ); } - let undo_results = join_all(futures).await; - let undo_error_count = undo_results - .iter() - .filter(|result| match result { - Err(_) | Ok(Err(_)) => true, - Ok(Ok(_)) => false, - }) - .count(); - if undo_error_count > 0 { - warn!( - target: "rustfs_ecstore::set_disk", - dst_bucket = %dst_bucket, - dst_object = %dst_object, - undo_error_count, - "rename_data quorum rollback reported errors" - ); - } + rollback_failed_rename( + disks, + file_infos, + &errs, + &dispatch_states, + &data_dirs, + (&dst_bucket, &dst_object), + rollback_receipt, + ) + .await; return Err(ret_err); } @@ -6800,6 +7086,86 @@ pub(in crate::set_disk) mod rename_fault_injection { } } +#[cfg(test)] +pub(in crate::set_disk) mod rollback_fault_injection { + use super::DiskError; + use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + }; + + #[derive(Clone, Copy, Debug)] + pub(in crate::set_disk) enum Fault { + Io, + Panic, + IoAfterRename, + VolumeNotFoundAfterRename, + PanicAfterRename, + CoordinatorPanic, + } + + fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Mutex::default) + } + + pub(in crate::set_disk) struct Guard(String); + + impl Drop for Guard { + fn drop(&mut self) { + if let Ok(mut registry) = registry().lock() { + registry.remove(&self.0); + } + } + } + + pub(in crate::set_disk) fn arm(object: &str, disk_index: usize, fault: Fault) -> Guard { + registry() + .lock() + .expect("rollback registry should not poison") + .insert(object.to_string(), (disk_index, fault)); + Guard(object.to_string()) + } + + pub(super) fn before_undo(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::Io)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::Panic)) if target == disk_index => panic!("injected rollback panic"), + _ => Ok(()), + } + } + + pub(super) fn after_rename(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::IoAfterRename)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::VolumeNotFoundAfterRename)) if target == disk_index => Err(DiskError::VolumeNotFound), + Some((target, Fault::PanicAfterRename)) if target == disk_index => panic!("injected panic after rename mutation"), + _ => Ok(()), + } + } + + pub(super) fn after_fanout(object: &str) { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + if matches!(fault, Some((_, Fault::CoordinatorPanic))) { + panic!("injected rename coordinator panic"); + } + } +} + /// Test-only per-disk call counters for the metadata fan-out (backlog#1325, /// serving the RPC-count assertions of #1309 / #1314 / #1315). /// @@ -6911,6 +7277,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { pub const RENAME: &str = "rename"; /// The per-disk old-data-dir cleanup phase of the commit fan-out. pub const CLEANUP: &str = "cleanup"; + pub const ROLLBACK: &str = "rollback"; /// The per-disk `read_version` phase of metadata read fan-out. #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub const READ_VERSION: &str = "read_version"; @@ -9668,6 +10035,7 @@ mod tests { file_info.mod_time = Some(OffsetDateTime::now_utc()); file_info.erasure.index = idx + 1; file_info.data = Some(Bytes::from_static(b"inline-body")); + file_info.set_inline_data(); file_info.metadata.insert("etag".to_string(), etag.to_string()); file_info }) @@ -10422,6 +10790,453 @@ mod tests { .await; } + #[tokio::test] + async fn rename_rollback_incomplete_inspection_rejection_is_not_recovery() { + fn reject_inspection(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + assert_eq!(request.bucket, "rollback-inspection"); + assert_eq!(request.object_prefix.as_deref(), Some("object")); + assert_eq!(request.dry_run, Some(true), "failed minority must never become a mutating heal source"); + assert_eq!(request.remove_corrupted, Some(false)); + assert_eq!(request.recreate_missing, Some(false)); + assert_eq!(request.recursive, Some(false)); + Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full) }) + } + let result = inspect_incomplete_rename_rollback(&[], "rollback-inspection", "object", reject_inspection).await; + assert!(matches!(result, ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full))); + } + + #[tokio::test] + async fn rename_rollback_incomplete_cancelled_task_is_not_success() { + let task = tokio::spawn(std::future::pending::>()); + task.abort(); + assert_eq!(rename_rollback_task_outcome(task.await), RenameRollbackOutcome::Cancelled); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + const DISKS: usize = 4; + const WRITE_QUORUM: usize = 3; + for overwrite in [false, true] { + for success_count in [0, WRITE_QUORUM - 1, WRITE_QUORUM] { + for fault in [ + None, + Some(rollback_fault_injection::Fault::Io), + Some(rollback_fault_injection::Fault::Panic), + ] { + let mut previous = None; + for early_ack in [false, true] { + let bucket = "rename-rollback-matrix"; + let object = format!("object-{overwrite}-{success_count}-{fault:?}-{early_ack}"); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + if overwrite { + let mut old = metadata_test_fileinfo(&object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old version must be staged"); + } + } + let _rename_fault = + rename_fault_injection::fail_rename_on(&object, &(success_count..DISKS).collect::>()); + let _undo_fault = fault.map(|fault| rollback_fault_injection::arm(&object, 0, fault)); + let receipt = RenameRollbackReceipt::default(); + let result = SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, DISKS, "new-etag"), + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(WRITE_QUORUM, None).with_rollback_receipt(receipt.clone()), + ) + .await; + let actual_error = match result { + Ok(commit) => { + assert_eq!(success_count, WRITE_QUORUM); + if let Some(tail) = commit.tail_drain { + tail.await + .expect("committed tail should join") + .expect("committed tail should converge"); + } + assert!( + receipt.0.get().is_none(), + "a quorum commit must not enter rollback even when undo faults are armed" + ); + None + } + Err(err) => { + assert!(success_count < WRITE_QUORUM); + let report = receipt + .0 + .get() + .expect("both failure paths must publish per-disk rollback evidence"); + assert_eq!(report.disks.len(), DISKS); + for (idx, outcome) in report.disks.iter().enumerate() { + assert_eq!(outcome.disk_index, idx); + let expected = if idx >= success_count { + RenameRollbackOutcome::NotAttempted(DiskError::other( + "injected rename failure (test-only)", + )) + } else if idx == 0 { + match fault { + Some(rollback_fault_injection::Fault::Io) => { + RenameRollbackOutcome::Failed(DiskError::FaultyDisk) + } + Some(rollback_fault_injection::Fault::Panic) => RenameRollbackOutcome::Panicked, + None => RenameRollbackOutcome::Succeeded, + Some(_) => unreachable!("matrix only injects undo faults"), + } + } else { + RenameRollbackOutcome::Succeeded + }; + assert_eq!(outcome.outcome, expected); + if overwrite && idx < success_count { + let backup = dirs[idx] + .path() + .join(bucket) + .join(&object) + .join(outcome.rollback_dir.expect("overwrite needs rollback dir").to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + outcome.outcome.needs_recovery(), + "failed undo must retain its only old-version backup" + ); + } + } + assert_eq!(receipt.is_incomplete(), success_count > 0 && fault.is_some()); + Some(err) + } + }; + if let Some(expected_error) = previous.as_ref() { + assert_eq!( + &actual_error, expected_error, + "early ACK must preserve the original full-wait quorum error" + ); + } + previous = Some(actual_error); + for (idx, dir) in dirs.iter().enumerate() { + let reopened = reopen_local_disk(dir).await; + let read = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await; + let keeps_new = + idx < success_count && (success_count == WRITE_QUORUM || (idx == 0 && fault.is_some())); + if keeps_new || overwrite { + let stored = read.expect("old or committed version must survive reopen"); + assert_eq!( + stored.metadata.get("etag").map(String::as_str), + Some(if keeps_new { "new-etag" } else { "old-etag" }) + ); + assert_eq!( + stored.data.as_deref(), + Some(if keeps_new { + b"inline-body".as_slice() + } else { + b"old-inline-body".as_slice() + }), + "object={object}, disk={idx}, keeps_new={keeps_new}" + ); + } else { + assert!( + matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)), + "fresh rollback must not expose data: {read:?}" + ); + } + } + } + } + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for fault in [ + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + ] { + let bucket = "rename-tail-unknown"; + let object = format!("tail-{fault:?}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let receipt = RenameRollbackReceipt::default(); + let _fault = rollback_fault_injection::arm(&object, 0, fault); + let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier_phase::RENAME); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, 4, "new-etag"), + (bucket, &object), + true, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("tail barrier must precede quorum ACK"), + } + }) + .await + .expect("tail reaches the barrier"); + let commit = tokio::time::timeout(BARRIER_PAUSE_GUARD, rename) + .await + .expect("quorum must ACK before tail release") + .expect("three disks commit"); + barrier.release(); + let tail = commit + .tail_drain + .expect("early ACK owns a tail") + .await + .expect("tail coordinator joins") + .expect("committed tail reports convergence"); + assert_eq!(tail.convergence, RenameConvergence::PartialCommit); + assert!(receipt.0.get().is_none(), "post-ACK errors must never start rollback"); + for dir in &dirs { + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("all actual writes survive despite a lost tail acknowledgement"); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + rollback_fault_injection::Fault::CoordinatorPanic, + ] { + for early_ack in [false, true] { + let bucket = "rollback-data-dirs"; + let object = format!("object-{early_ack}-{fault:?}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let old_data_dir = Uuid::new_v4(); + let new_data_dir = Uuid::new_v4(); + let mut old = metadata_test_fileinfo(&object); + old.data_dir = Some(old_data_dir); + old.mod_time = Some(OffsetDateTime::now_utc()); + let mut infos = Vec::new(); + for (idx, disk) in disks.iter().enumerate() { + let disk = disk.as_ref().expect("fixture disk should be present"); + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old metadata should be staged"); + let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string()); + tokio::fs::create_dir_all(&old_dir) + .await + .expect("old data directory should exist"); + tokio::fs::write(old_dir.join("part.1"), b"old-data") + .await + .expect("old shard should exist"); + let source = dirs[idx] + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()); + tokio::fs::create_dir_all(&source) + .await + .expect("new data directory should be staged"); + tokio::fs::write(source.join("part.1"), b"new-data") + .await + .expect("new shard should be staged"); + let mut fi = metadata_test_fileinfo(&object); + fi.data_dir = Some(new_data_dir); + fi.erasure.index = idx + 1; + fi.mod_time = Some(OffsetDateTime::now_utc()); + infos.push(fi); + } + let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(&object, 0, fault); + let receipt = RenameRollbackReceipt::default(); + assert!( + SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + ) + .await + .is_err() + ); + assert!(receipt.is_incomplete()); + if !matches!(fault, rollback_fault_injection::Fault::Io) { + assert!( + matches!( + receipt.0.get().expect("indeterminate report").disks[0].outcome, + RenameRollbackOutcome::Indeterminate(_) + ), + "post-mutation failure must not be classified as unattempted" + ); + } + let coordinator_failed = matches!(fault, rollback_fault_injection::Fault::CoordinatorPanic); + for (idx, dir) in dirs.iter().enumerate() { + let root = dir.path().join(bucket).join(&object); + assert_eq!( + tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1")) + .await + .expect("old data must survive failed overwrite"), + b"old-data" + ); + let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + idx == 0 || (coordinator_failed && idx == 1), + "unknown mutations must retain the old-version backup" + ); + if idx >= 2 { + let staged = dir + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()) + .join("part.1"); + assert_eq!( + tokio::fs::read(staged) + .await + .expect("failed-write staging must remain available"), + b"new-data" + ); + } + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("metadata should survive reopen"); + assert_eq!( + stored.data_dir, + Some(if idx == 0 || (coordinator_failed && idx == 1) { + new_data_dir + } else { + old_data_dir + }) + ); + } + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { + for cancel_caller in [false, true] { + let bucket = "rename-rollback-barrier"; + let object = if cancel_caller { + "rollback-barrier-cancelled" + } else { + "rollback-barrier-object" + }; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("old metadata should be staged"); + } + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + false, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + } + }) + .await + .expect("undo must reach its disk barrier"); + assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); + if cancel_caller { + drop(rename); + barrier.release(); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while receipt.0.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled caller must not cancel rollback accounting"); + } else { + barrier.release(); + assert!(rename.await.is_err()); + } + assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + for dir in dirs.iter().skip(1) { + let reopened = reopen_local_disk(dir).await; + let restored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("old version must remain readable after caller cancellation"); + assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } + } + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_data_early_ack_strict_quorum_failure_rolls_back_fresh_after_reopen() { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 1bc87619d..0690d70cb 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; pub(crate) use ops::object::DeleteObjectCommitBarrier; #[cfg(any(test, feature = "test-util"))] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier; pub(crate) use ops::object::body_cache_plaintext_len; #[cfg(all(test, feature = "test-util"))] diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 5a220e432..c76daf707 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{ NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, - SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, + SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion, }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; -use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; +use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; #[cfg(test)] @@ -3548,6 +3548,7 @@ impl SetDisks { (None, None, None) }; let mut tmp_cleanup_owned = false; + let rollback_receipt = RenameRollbackReceipt::default(); let operation = async { let erasure = Arc::new(erasure_from_file_info(&fi, false)?); @@ -4256,6 +4257,7 @@ impl SetDisks { let commit_bucket = bucket.to_owned(); let commit_object = object.to_owned(); let commit_tmp_dir = tmp_dir.clone(); + let commit_rollback_receipt = rollback_receipt.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); let commit_publication_guard = publication_commit_guard.take(); @@ -4266,13 +4268,17 @@ impl SetDisks { // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases // remote leases. - let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) - && (commit_object_lock_guard.is_some() - || commit_decommission_object_lock_guard.is_some() - || commit_publication_guard.is_some()) + let commit_owns_namespace_guard = commit_object_lock_guard.is_some() + || commit_decommission_object_lock_guard.is_some() + || commit_publication_guard.is_some(); + let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum + && !(opts.data_movement && opts.has_decommission_capacity_reservation()) + && commit_owns_namespace_guard && commit_scanner_publication_scope.is_none(); + // Full-tail callers also transfer owned guards to the coordinator: + // cancelling their ACK waiter must not cancel an in-flight rename. let detach_commit_owner = commit_scanner_publication_scope.is_some() - || commit_allows_early_ack + || commit_owns_namespace_guard || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence; let commit_write_path_label = write_path.metric_label(); @@ -4452,7 +4458,8 @@ impl SetDisks { write_quorum, commit_scanner_publication_lease_tokens.as_ref(), ) - .with_publication_scope(commit_scanner_publication_scope.clone()), + .with_publication_scope(commit_scanner_publication_scope.clone()) + .with_rollback_receipt(commit_rollback_receipt.clone()), ) .await; if let Some(scope) = commit_scanner_publication_scope.as_ref() { @@ -4585,6 +4592,11 @@ impl SetDisks { let rename_commit = match rename_result { Ok(commit) => commit, Err(err) => { + if commit_rollback_receipt.is_incomplete() { + // Incomplete undo retains the staging source and + // rollback backup for recovery; cleanup is unsafe. + return Err(err.into()); + } if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -4617,9 +4629,8 @@ impl SetDisks { request.object_version_id = committed_version_id .or_else(|| commit_version_suspended.then(Uuid::nil)) .map(|version_id| version_id.to_string()); - tokio::spawn(async move { - let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await; - }); + let heal_set = commit_set.clone(); + tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await }); } let rename_stage_elapsed = rename_stage_start.elapsed(); @@ -4885,7 +4896,7 @@ impl SetDisks { ); } }); - } else { + } else if !rollback_receipt.is_incomplete() { // Failure path (quorum loss / rollback): keep the cleanup inline so // a failed PUT never returns while its tmp shards are still on disk // (state-residue hardening tracked by backlog#864 / backlog#898). @@ -17494,27 +17505,69 @@ mod put_object_tmp_cleanup_tests { } #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] async fn put_object_failure_cleans_tmp_workspace_inline() { - let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "tmp-clean-missing-bucket"; + let object = "orphan-object"; + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let writer = Arc::clone(&set_disks); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("missing-bucket PUT must stage before rename"); + let staged = non_trash_tmp_entries(&temp_dirs).await; + assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection"); + for workspace in staged { + let mut entries = tokio::fs::read_dir(&workspace) + .await + .expect("staged workspace should be readable"); + let mut shards = 0; + while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") { + if entry.file_type().await.expect("staged entry type").is_dir() { + let part = tokio::fs::metadata(entry.path().join("part.1")) + .await + .expect("staging must contain an actual erasure shard"); + assert!(part.len() > 0, "the shard must be written before the missing-bucket failure"); + shards += 1; + } + } + assert_eq!(shards, 1); + } + assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists())); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("missing-bucket PUT must finish") + .expect("PUT task should join") + .expect_err("put_object into a missing bucket volume must fail"); + assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}"); - // The bucket volume is never created, so the shards are written into - // the tmp workspace and the commit fails at rename_data with a quorum - // error — exercising the failure-path cleanup. - let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); - let err = set_disks - .put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default()) - .await - .expect_err("put_object into a missing bucket volume must fail"); - - // No polling: the failure path must clean the tmp workspace inline, - // before put_object returns (backlog#864 / backlog#898 hardening). - let leftovers = non_trash_tmp_entries(&temp_dirs).await; - assert!( - leftovers.is_empty(), - "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" - ); - - drop(temp_dirs); + // No polling: known pre-publication rejection must clean staging + // inline, before PUT returns (backlog#864 / backlog#898). + let leftovers = non_trash_tmp_entries(&temp_dirs).await; + assert!( + leftovers.is_empty(), + "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" + ); + } + }) + .await; } #[tokio::test] @@ -18157,6 +18210,354 @@ mod put_object_tmp_cleanup_tests { .await; } + async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) { + for disk in disks { + disk.make_volume(bucket) + .await + .expect("completion test bucket should be created"); + } + } + + /// Observe the actual metadata quorum while the remaining rename is parked. + /// A completed task count alone can race tasks that have not started yet. + async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let mut committed = 0; + for disk in disks { + match disk.read_version("", bucket, object, "", &ReadOptions::default()).await { + Ok(_) => committed += 1, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {} + Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"), + } + } + if committed == 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("three disks must publish metadata while the fourth rename remains paused"); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for size in [4096, 1024 * 1024] { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-cas"; + let object = "full-tail-cas-object"; + make_completion_test_bucket(&disks, bucket).await; + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; size]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("full-tail PUT must reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum"); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "the owned namespace guard must remain held" + ); + barrier.release(); + let written = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("full-tail PUT should finish after release") + .expect("full-tail PUT task should join") + .expect("full-tail PUT must commit"); + assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task"); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("same-key lock should be available on return") + .expect("same-key lock probe should succeed"), + ); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("successful full-tail PUT must publish on every healthy disk"); + } + drop(barrier); + let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec()); + set.put_object( + bucket, + object, + &mut replacement, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: written.etag, + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect("immediate same-key CAS must acquire the namespace guard"); + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("CAS successor must be immediately readable"); + let mut body = Vec::new(); + read.stream.read_to_end(&mut body).await.expect("successor body must drain"); + assert_eq!(body, b"cas successor"); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-heal"; + let object = "full-tail-heal-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut heals = set.capture_test_rename_tail_heals(); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _fault = rename_fault_injection::fail_rename_on(object, &[0]); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("failed tail must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "committed quorum must still wait for the failing tail"); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("failed tail should drain") + .expect("PUT task should join") + .expect("a minority tail error must not negate committed quorum"); + assert_eq!(tasks.running(), 0); + let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv()) + .await + .expect("failed tail must schedule heal") + .expect("heal capture must remain connected"); + assert_eq!(heal.bucket, bucket); + assert_eq!(heal.object_prefix.as_deref(), Some(object)); + let info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("committed object must remain readable despite the failed tail"); + assert_eq!(info.size, TEST_OBJECT_SIZE as i64); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_rejects_quorum_minus_one() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-no-quorum"; + let object = "full-tail-no-quorum-object"; + make_completion_test_bucket(&disks, bucket).await; + let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]); + let tasks = rename_fanout_barrier::observe_tasks(object); + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + let err = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect_err("draining two successful disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return"); + assert!( + set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(), + "failed fresh write must not become visible" + ); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() { + use crate::set_disk::core::io_primitives::rollback_fault_injection; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + ] { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-incomplete-undo"; + let object = "incomplete-undo-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]); + set.put_object( + bucket, + object, + &mut old_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("old generation should be completely committed"); + wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await; + let old = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("old metadata must be readable"); + let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, fault); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("overwrite must enter the actual rename fan-out before failure injection"); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("incomplete undo must return without hanging") + .expect("PUT task should join") + .expect_err("two renamed disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return"); + let leftovers = non_trash_tmp_entries(&dirs).await; + assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery"); + let backups = dirs + .iter() + .filter(|dir| { + dir.path() + .join(bucket) + .join(object) + .join(old_data_dir.to_string()) + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + .exists() + }) + .count(); + assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup"); + // The remaining three disks still serve the old generation; + // the failed minority must never become an acknowledged write. + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("old generation must remain readable after incomplete rollback"); + let mut body = Vec::new(); + read.stream + .read_to_end(&mut body) + .await + .expect("old generation should stream"); + assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_owned_commit_survives_waiter_cancellation() { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = RUSTFS_META_BUCKET; + let object = "full-tail-cancelled-receipt"; + // Internal config writes do not own a bucket lifecycle guard. The object + // guard alone must keep the full-tail coordinator alive after cancellation. + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("cancelled receipt must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + put.abort(); + assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled()); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "owned coordinator must retain the namespace guard after waiter cancellation" + ); + barrier.release(); + drop( + tokio::time::timeout(Duration::from_secs(30), lock_probe) + .await + .expect("cancelled coordinator must eventually release its guard") + .expect("post-commit lock probe should succeed"), + ); + assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task"); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("caller cancellation must not interrupt committed receipt materialization"); + } + wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn no_lock_put_waits_for_rename_tail_under_outer_guard() { @@ -18184,6 +18585,7 @@ mod put_object_tmp_cleanup_tests { &mut reader, &ObjectOptions { no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -18209,7 +18611,18 @@ mod put_object_tmp_cleanup_tests { put.await .expect("no-lock PUT task should join") .expect("no-lock PUT should commit after the rename tail releases"); + let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "full-tail PUT must not release the caller's outer guard" + ); drop(outer_guard); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("outer owner releasing its guard should unblock the probe") + .expect("post-outer-guard probe should succeed"), + ); }) .await; } diff --git a/crates/ecstore/src/set_disk/transition_matrix_tests.rs b/crates/ecstore/src/set_disk/transition_matrix_tests.rs index 8924b2a4a..9b5912ce7 100644 --- a/crates/ecstore/src/set_disk/transition_matrix_tests.rs +++ b/crates/ecstore/src/set_disk/transition_matrix_tests.rs @@ -18,6 +18,7 @@ use super::{ }; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time}; use crate::ecstore_validation_blackbox::make_local_set_disks; +use crate::object_api::WriteCompletion; use crate::services::tier::test_util::register_mock_tier; use crate::storage_api_contracts::bucket::BucketOperations; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() { object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 22d7e37c3..d0b426b68 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -2979,6 +2979,33 @@ mod tests { #[cfg(feature = "test-util")] const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object"; + fn decommission_retry_fault_hook( + bucket: &str, + object: &str, + faults: Arc, + ) -> crate::core::pools::DecommissionTestFaultDecision { + let target_bucket = bucket.to_string(); + let target_object = object.to_string(); + Arc::new(move |stage, bucket, object, _attempt, succeeded| { + if !succeeded + || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT + || bucket != target_bucket + || object != target_object + { + return false; + } + + // Entry retries reset the local attempt; real copy errors can skip + // successful attempts. Only injected faults spend this global budget. + faults + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| { + (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)) + .then_some(faults.saturating_add(1)) + }) + .is_ok() + }) + } + async fn seed_decommission_source( store: &Arc, bucket: &str, @@ -5120,6 +5147,33 @@ mod tests { shutdown.cancel(); } + #[test] + fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() { + for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] { + let faults = Arc::new(AtomicUsize::new(0)); + let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults)); + + for (stage, bucket, object, succeeded) in [ + ("other-stage", "bucket", "object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true), + (DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false), + ] { + assert!(!hook(stage, bucket, object, 1, succeeded)); + } + assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults"); + + for (index, attempt) in attempts.into_iter().enumerate() { + assert_eq!( + hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true), + index < 2, + "attempts={attempts:?}, index={index}" + ); + } + assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}"); + } + } + #[test] #[serial_test::serial(storage_class_env)] fn decommission_entry_retries_source_changed_without_canceling_other_bucket() { @@ -5214,31 +5268,8 @@ mod tests { )); let ordinary_faults = Arc::new(AtomicUsize::new(0)); - let ordinary_faults_for_hook = Arc::clone(&ordinary_faults); - let fault_bucket = other_bucket.clone(); - let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new( - move |stage, bucket, object, attempt, succeeded| { - let candidate = succeeded - && stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT - && bucket == fault_bucket.as_str() - && object == other_object; - if !candidate { - return false; - } - - // Keep the fault budget global across any - // entry-level re-list; its inner attempt counter - // restarts after SourceChanged. - ordinary_faults_for_hook - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| { - let next_fault = faults.saturating_add(1); - (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1) - && attempt == next_fault) - .then_some(next_fault) - }) - .is_ok() - }, - )); + let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults)); + let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook); let rx = CancellationToken::new(); let source_changed_exhaustions = Arc::new(AtomicUsize::new(0)); @@ -8045,10 +8076,15 @@ mod tests { ); assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) + let full_tail = ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail) .await .expect("second page receipt should restore"); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail) .await .expect("second page receipt should corrupt deterministically"); let corrupt = store @@ -11575,6 +11611,7 @@ mod tests { pool_index: usize, bucket: &str, object: &str, + minio_unversioned: bool, ) { for disk_index in 0..4 { let metadata_path = @@ -11608,6 +11645,11 @@ mod tests { ] { rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix); } + if minio_unversioned { + object_meta + .meta_sys + .insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new()); + } *shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version) .expect("legacy transitioned version should re-encode"); } @@ -11618,6 +11660,152 @@ mod tests { } } + #[cfg(feature = "test-util")] + async fn read_store_body( + store: &Arc, + bucket: &str, + object: &str, + range: Option, + opts: &ObjectOptions, + ) -> Vec { + let mut reader = store + .get_object_reader(bucket, object, range, HeaderMap::new(), opts) + .await + .expect("object reader should open"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("object body should drain"); + body + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() { + let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_put_remote_version(Some(String::new())).await; + let bucket = "legacy-unknown-unversioned-read-bucket"; + let object = "object.bin"; + let payload = b"legacy unversioned remote tier object remains readable".repeat(1024); + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("legacy source bucket should be created"); + let mut reader = PutObjReader::from_vec(payload.clone()); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("legacy source should be written"); + store + .transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("legacy source should have an etag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ) + .await + .expect("legacy source should transition"); + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await; + backend.clear_op_log().await; + + let opts = ObjectOptions { + metadata_cache_safe: false, + ..Default::default() + }; + let head = store + .get_object_info(bucket, object, &opts) + .await + .expect("legacy transitioned HEAD should use local metadata"); + assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); + assert!(head.transitioned_object.version_id.is_empty()); + assert_eq!( + head.user_defined + .get("x-minio-internal-transitioned-versionID") + .map(String::as_str), + Some(""), + "the MinIO empty version-key provenance must survive xl.meta decoding" + ); + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str( + &head.user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ), + "the compatibility read must not synthesize version-state metadata" + ); + + let full_body = read_store_body(&store, bucket, object, None, &opts).await; + assert_eq!(full_body, payload); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 7, + end: 38, + }; + let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await; + assert_eq!(ranged_body, &payload[7..=38]); + + let after_read = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("legacy metadata should remain readable after GET") + .expect("legacy object metadata should remain on disk") + .versions + .into_iter() + .find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE) + .expect("legacy transitioned source should remain visible after GET"); + assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); + assert!(after_read.transition_version.is_none()); + assert!(after_read.transition_version_id.is_none()); + assert_eq!( + after_read + .metadata + .get("x-minio-internal-transitioned-versionID") + .map(String::as_str), + Some(""), + "the MinIO empty version-key provenance must remain after GET and Range GET" + ); + assert!( + !rustfs_utils::http::metadata_compat::contains_key_str( + &after_read.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ), + "the compatibility read must remain side-effect free" + ); + + assert_eq!( + backend.op_log().await, + vec![ + MockWarmOp::Probe { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Get { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Probe { + object: after_read.transitioned_objname.clone(), + }, + MockWarmOp::Get { + object: after_read.transitioned_objname, + }, + ], + "legacy reads should probe before each unversioned GET and never mutate local metadata" + ); + assert_eq!(backend.remove_count().await, 0); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -11658,7 +11846,7 @@ mod tests { ) .await .expect("legacy source should transition"); - rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await; + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await; let legacy = store.pools[0] .get_disks_by_key(object) .load_file_info_versions_exact(bucket, object) @@ -12799,7 +12987,7 @@ mod tests { .expect("merge-loser source should transition"); copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await; } - rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await; + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await; backend.set_remove_failure(true); store.pools[1] .delete_object(bucket, "hidden/item.bin", ObjectOptions::default()) @@ -16866,6 +17054,10 @@ mod tests { .find(|version| version.version_id == history.version_id) .expect("transitioned history should exist"); transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; + rustfs_utils::http::metadata_compat::remove_str( + &mut transitioned.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, + ); metadata .add_version(transitioned) .expect("unknown state should replace the transitioned version"); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 77aaf98c8..8a4579e1b 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -425,7 +425,7 @@ pub(crate) mod init_format; pub(crate) mod list_objects; mod multipart; mod object; -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub use object::DeleteAfterObjectLockSnapshotBarrier; pub(crate) use object::{ DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence, diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index f0969c3cf..42aa5a74d 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -297,6 +297,20 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio } } +fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String { + decoded.map(str::to_owned).unwrap_or_else(|| { + if raw.is_empty() { + String::new() + } else { + String::from_utf8_lossy(raw).into_owned() + } + }) +} + +fn is_transition_version_metadata_key(key: &str) -> bool { + strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID)) +} + fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> { let valid = match state { TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(), @@ -366,14 +380,26 @@ impl<'a> DerivedInternalMetadata<'a> { } *slot = Some(value.as_slice()); } + fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result> { + if let (Some(canonical), Some(legacy)) = (canonical, legacy) + && canonical != legacy + { + return Err(Error::FileCorrupt); + } + Ok(canonical.or(legacy)) + } + Ok(Self { checksum: canonical.checksum.or(legacy.checksum), part_checksums: canonical.part_checksums.or(legacy.part_checksums), - transition_status: canonical.transition_status.or(legacy.transition_status), - transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object), - transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version), - transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state), - transition_tier: canonical.transition_tier.or(legacy.transition_tier), + transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?, + transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?, + transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?, + transitioned_version_state: merge_consistent( + canonical.transitioned_version_state, + legacy.transitioned_version_state, + )?, + transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?, }) } } @@ -438,8 +464,14 @@ impl FileInfo { } } -fn set_transition_version_state(meta_sys: &mut HashMap>, state: TransitionVersionState) { - if state == TransitionVersionState::Unknown { +fn set_transition_version_state( + meta_sys: &mut HashMap>, + state: TransitionVersionState, + source_metadata: &HashMap, +) { + if state == TransitionVersionState::Unknown + && !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE) + { remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE); } else { insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec()); @@ -2643,6 +2675,11 @@ impl MetaObject { if derived_metadata.transitioned_version_state.is_some() { validate_transition_version_state(transition_version_state, transition_version.as_deref())?; } + for (key, value) in &self.meta_sys { + if is_transition_version_metadata_key(key) { + metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref())); + } + } let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); let transition_tier = derived_metadata .transition_tier @@ -2689,7 +2726,7 @@ impl MetaObject { } else { remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); } - set_transition_version_state(&mut self.meta_sys, fi.transition_version_state); + set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata); insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) { insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes()); @@ -2830,7 +2867,7 @@ impl From for MetaObject { insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); } if !value.transition_status.is_empty() { - set_transition_version_state(&mut meta_sys, value.transition_version_state); + set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata); } if !value.transition_tier.is_empty() { @@ -2985,6 +3022,12 @@ impl MetaDeleteMarker { fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?; fi.transition_version = transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state); + for (key, value) in &self.meta_sys { + if is_transition_version_metadata_key(key) { + fi.metadata + .insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref())); + } + } fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); if derived_metadata.transitioned_version_state.is_some() { validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?; @@ -3152,7 +3195,7 @@ impl From for MetaDeleteMarker { insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); } if !value.transition_status.is_empty() || value.tier_free_version() { - set_transition_version_state(&mut meta_sys, value.transition_version_state); + set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata); } if !value.transition_tier.is_empty() { insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec()); @@ -4574,6 +4617,7 @@ mod tests { .into_fileinfo("b", "k", false) .expect("into_fileinfo"); assert_eq!(fi.transition_version_id, None); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new())); } #[test] @@ -4585,6 +4629,10 @@ mod tests { .into_fileinfo("b", "k", false) .expect("into_fileinfo"); assert_eq!(fi.transition_version_id, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "nil UUID bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4598,6 +4646,7 @@ mod tests { assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string())); } #[test] @@ -4637,6 +4686,36 @@ mod tests { assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); } + #[test] + fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() { + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + SUFFIX_TRANSITIONED_VERSION_STATE, + TransitionVersionState::Unknown.as_str().to_string(), + ); + let fi = FileInfo { + transition_status: "complete".to_string(), + transition_version_state: TransitionVersionState::Unknown, + metadata, + ..Default::default() + }; + + let object = MetaObject::from(fi); + assert_eq!( + get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE), + Some(b"unknown".as_slice()) + ); + let decoded = object + .into_fileinfo("b", "k", false) + .expect("explicit unknown state should decode"); + assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown); + assert_eq!( + rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,), + Some("unknown") + ); + } + #[test] fn meta_object_transition_version_state_exact_round_trips_dual_keys() { let id = sample_version_id(); @@ -4753,6 +4832,10 @@ mod tests { .expect("invalid transition version bytes must not fail the object read"); assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "invalid raw bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4795,6 +4878,10 @@ mod tests { .into_fileinfo("b", "k", false) .expect("nil tier version should remain an absent remote version"); assert_eq!(fi.transition_version_id, None); + assert!( + get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()), + "nil UUID bytes must remain distinguishable from an empty MinIO version" + ); } #[test] @@ -4812,6 +4899,7 @@ mod tests { .expect("legacy binary UUID tier version should decode"); assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); + assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string())); } #[test] @@ -4910,6 +4998,23 @@ mod tests { assert_eq!(err, Error::FileCorrupt); } + #[test] + fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() { + let sys = HashMap::from([ + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), + b"unknown".to_vec(), + ), + ("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()), + ]); + + let err = make_meta_object_with_sys(sys) + .into_fileinfo("b", "k", false) + .expect_err("mixed-case transition state aliases must agree"); + + assert_eq!(err, Error::FileCorrupt); + } + #[test] fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() { let object = FileMetaVersionHeader { diff --git a/crates/scanner/src/remote_scanner.rs b/crates/scanner/src/remote_scanner.rs index 42ee47c52..47ba5eacd 100644 --- a/crates/scanner/src/remote_scanner.rs +++ b/crates/scanner/src/remote_scanner.rs @@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock> = LazyLock::new(|| mod stream; +#[cfg(test)] +pub(crate) use stream::checkpoint_fixture_partial_return; + pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request}; pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket}; use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle}; diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 7d128cdd4..26f3610b6 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream( #[cfg(test)] const TEST_NEXT_CYCLE: u64 = 11; +#[cfg(test)] +pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let mut bytes = Vec::new(); + write_frame( + &mut bytes, + &writer_auth, + &mut 0, + &RemoteScannerFrame::terminal( + RemoteScannerProgress { + objects_scanned: progress.0, + directories_started: progress.1, + entries_visited, + }, + RemoteScannerFrameResult::Partial, + ), + ) + .await + .expect("checkpoint partial frame must encode"); + let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0) + .await + .expect("checkpoint progress frame must authenticate"); + assert_eq!(frame.progress.entries_visited, entries_visited); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + let result = consume_remote_scanner_stream( + std::io::Cursor::new(bytes), + parent, + budget.clone(), + "bucket", + DataUsageCacheSource::new(0, 0), + DataUsageScanPlanDigest([17; 32]), + reader_auth, + ) + .await + .expect("checkpoint partial frame must decode"); + assert!(matches!(result, RemoteScannerOutcome::Partial)); + assert_eq!(budget.progress(), progress); +} + #[cfg(test)] async fn consume_remote_scanner_stream( reader: R, diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index ade37c88c..650ccaab3 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -24,6 +24,8 @@ use std::io::Write; use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; +mod checkpoint_fixture; + /// Reset the process-global alert cooldown map; test-only. fn reset_alert_cooldowns() { *SCANNER_ALERT_EMISSION_COOLDOWN diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs new file mode 100644 index 000000000..466383993 --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -0,0 +1,410 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::scanner_budget::ScannerCycleBudgetConfig; +use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk}; +use crate::storage_api::scanner_io::ObjectIO; +use crate::{DataUsageCacheSource, DataUsageScanPlanDigest}; +use std::io::Cursor; +use tokio::io::AsyncReadExt; + +const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; +const STATIC_OBJECTS: u64 = 24; +const MAX_CACHE_BYTES: u64 = 1024 * 1024; +const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0); +const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]); + +/// Real cache persistence codec and CAS calls, backed by two bounded local files. +#[derive(Debug)] +struct FixtureStore { + root: tempfile::TempDir, + reject_save: AtomicBool, +} + +impl FixtureStore { + fn new() -> Arc { + Arc::new(Self { + root: tempfile::tempdir().expect("checkpoint fixture storage directory"), + reject_save: AtomicBool::new(false), + }) + } + + fn path(&self, object: &str) -> std::path::PathBuf { + assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp"))); + self.root + .path() + .join(if object.ends_with(".bkp") { "backup" } else { "main" }) + } + + async fn strict_load(&self) -> DataUsageCache { + let bytes = tokio::fs::read(self.root.path().join("main")) + .await + .expect("saved checkpoint fixture must exist"); + decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root") + } +} + +#[async_trait::async_trait] +impl ObjectIO for FixtureStore { + type Error = crate::EcstoreError; + type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + _bucket: &str, + object: &str, + _range: Option, + _headers: Self::HeaderMap, + _options: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + crate::EcstoreError::FileNotFound + } else { + crate::EcstoreError::from(error) + } + })?; + assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); + Ok(crate::ScannerGetObjectReader { + stream: Box::new(Cursor::new(bytes)), + object_info: crate::ScannerObjectInfo { + etag: Some("fixture".into()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + _bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + options: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + if self.reject_save.load(Ordering::SeqCst) { + return Err(crate::EcstoreError::PreconditionFailed); + } + let path = self.path(object); + let exists = tokio::fs::try_exists(&path).await?; + let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS"); + if (exists && preconditions.if_none_match_value() == Some("*")) + || (!exists && preconditions.if_match_value().is_some()) + || (exists && preconditions.if_match_value() != Some("fixture")) + { + return Err(crate::EcstoreError::PreconditionFailed); + } + let mut bytes = Vec::new(); + (&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?; + assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES); + tokio::fs::write(path, bytes).await?; + Ok(crate::ScannerObjectInfo { + etag: Some("fixture".into()), + ..Default::default() + }) + } +} + +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for FixtureStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _options: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + +fn decode_fixture(bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") { + return Err("missing or oversized checkpoint fixture"); + } + let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?; + if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() { + return Err("checkpoint fixture has no valid bucket root"); + } + Ok(cache) +} + +fn retained(cache: &DataUsageCache) -> u64 { + assert!( + !cache.root().is_some_and(|root| root.compacted), + "a compacted bucket root cannot prove static-prefix coverage" + ); + cache + .checked_flatten("bucket/static") + .map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64")) +} + +#[derive(Debug, PartialEq, Eq)] +enum CoverageDiagnosis { + Progress, + NoNewWork, + LostAtPrepare, + LostAtReload, + WalkWithoutRetention, +} + +fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis { + if reloaded < scanned { + CoverageDiagnosis::LostAtReload + } else if prepared < previous { + CoverageDiagnosis::LostAtPrepare + } else if walked > 0 && reloaded <= previous { + CoverageDiagnosis::WalkWithoutRetention + } else if reloaded > previous { + CoverageDiagnosis::Progress + } else { + CoverageDiagnosis::NoNewWork + } +} + +#[test] +fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() { + assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress); + assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention); + assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare); + assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload); + assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork); +} + +#[test] +fn checkpoint_fixture_missing_and_corrupt_inputs_fail() { + for bytes in [ + vec![], + vec![0xc1], + DataUsageCache::default().marshal_msg().expect("empty cache encoding"), + vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")], + ] { + assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root"); + } +} + +#[test] +fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() { + let mut cache = DataUsageCache::default(); + cache.info.name = "bucket".to_string(); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace("bucket/static", "bucket", DataUsageEntry::default()); + for index in 0..4 { + cache.replace( + &format!("bucket/static/{index}"), + "bucket/static", + DataUsageEntry { + objects: 1, + ..Default::default() + }, + ); + } + cache.reduce_children_of(&hash_path("bucket/static"), 1, true); + let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture"); + let entry = decoded + .find("bucket/static") + .expect("compaction must retain the static subtree root"); + assert!(entry.compacted); + assert!(entry.children.is_empty()); + assert_eq!( + retained(&decoded), + 4, + "compaction retains aggregate coverage even when leaf keys are absent" + ); +} + +#[tokio::test] +#[serial] +async fn checkpoint_fixture_save_reload_resume() { + run_checkpoint_fixture(false).await; +} + +#[tokio::test] +#[serial] +async fn checkpoint_fixture_hot_digest_diagnostic() { + run_checkpoint_fixture(true).await; +} + +async fn run_checkpoint_fixture(change_digest: bool) { + let (scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for index in 0..STATIC_OBJECTS { + write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await; + } + let store = FixtureStore::new(); + let mut previous = 0; + let mut visited = 0; + for round in 0..3_u8 { + write_test_object_metadata(&root, "bucket", "hot/current").await; + let mut cache = DataUsageCache::default(); + let revisions = cache + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("load checkpoint revisions"); + if round > 0 { + assert_eq!(retained(&store.strict_load().await), previous); + } + let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round))); + crate::scanner_io::current_cache_root_or_prepare_with_generation( + &mut cache, + "bucket", + SOURCE, + 11, + 7, + plan, + crate::scanner_io::DataUsageCacheReuseOptions { + require_source: true, + tier_registry_generation: None, + }, + ); + let prepared = retained(&cache); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_objects: Some(4), + ..Default::default() + }, + ); + let outcome = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget.clone(), + vec![scanner.local_disk.clone()], + cache, + None, + HealScanMode::Normal, + ) + .await + .expect("budgeted local disk scan returns partial cache"); + let ScannerDiskScanOutcome::Partial(cache) = outcome else { + panic!("budgeted fixture must remain partial") + }; + assert!(!cache.info.snapshot_complete, "partial must never publish a complete root"); + assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); + let scanned = retained(&cache); + cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("persist partial checkpoint"); + let mut loaded = DataUsageCache::default(); + loaded + .load(store.clone(), CACHE_NAME) + .await + .expect("reload persisted partial checkpoint"); + let reloaded = retained(&loaded); + assert_eq!(reloaded, retained(&store.strict_load().await)); + assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage"); + assert!(!loaded.info.snapshot_complete); + visited += budget.entries_visited(); + let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded); + eprintln!( + "checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}" + ); + if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() { + assert_eq!( + diagnosis, + CoverageDiagnosis::Progress, + "visited growth must produce durable static coverage" + ); + } + crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await; + previous = reloaded; + } + assert!(visited > 0, "fixture must exercise the directory walk"); + assert!(previous > 0, "fixture must retain and enumerate static subtree entries"); + + let mut loaded = DataUsageCache::default(); + let revisions = loaded + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("load final checkpoint"); + let before = tokio::fs::read(store.root.path().join("main")) + .await + .expect("read durable checkpoint bytes"); + let epoch_error = loaded + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1) + .await + .expect_err("stale publication epoch must reject persistence"); + assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED)); + store.reject_save.store(true, Ordering::SeqCst); + loaded.info.next_cycle += 1; + loaded + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect_err("injected save failure must not report durable progress"); + assert_eq!( + tokio::fs::read(store.root.path().join("main")) + .await + .expect("read unchanged checkpoint bytes"), + before + ); + + let parent = CancellationToken::new(); + parent.cancel(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let result = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget.clone(), + vec![scanner.local_disk.clone()], + loaded.clone(), + None, + HealScanMode::Normal, + ) + .await; + assert!(result.is_err(), "pre-scan cancellation must not produce a complete root"); + assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion"); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, Default::default()); + let result = scanner + .local_disk + .clone() + .nsscanner_disk( + budget.token(), + budget, + vec![scanner.local_disk.clone()], + loaded, + None, + HealScanMode::Normal, + ) + .await + .expect("unbounded scan must complete after durable partial progress"); + let ScannerDiskScanOutcome::Complete(cache) = result else { + panic!("unbounded fixture must produce a complete disk cache"); + }; + assert!(cache.info.snapshot_complete); + assert!(cache.info.scan_checkpoint.is_none()); + assert_eq!( + cache.checked_flatten("bucket").expect("complete bucket root").objects, + usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize") + ); +} diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index f568050d8..4807e11b8 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -209,6 +209,14 @@ fn scanner_bucket_cache_digest( DataUsageScanPlanDigest(hasher.finalize().into()) } +#[cfg(test)] +pub(crate) fn checkpoint_fixture_bucket_digest( + scan_plan_digest: DataUsageScanPlanDigest, + dirty_generation: Option, +) -> DataUsageScanPlanDigest { + scanner_bucket_cache_digest(scan_plan_digest, dirty_generation) +} + fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option) -> Result<()> { if results.iter().any(|result| result.info.last_update.is_some()) { return Ok(()); diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index a59d620fc..6fddafdfe 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -1048,6 +1048,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() { } } +#[test] +fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() { + for (budget, cancelled, bucket, expected) in [ + (false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded), + (true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete), + (false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete), + ] { + assert_eq!( + classify_nsscanner_cycle( + true, + budget, + cancelled, + bucket, + DirtyUsageSnapshotStatus::Changed, + ScannerCycleActivityStatus::Unchanged + ), + expected, + ); + } +} + #[test] fn unverified_activity_defers_partial_and_floor_cycles() { let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable); diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 5558ffa44..312d2928d 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -49,5 +49,14 @@ rustfs-rio.workspace = true tokio = { workspace = true, features = ["io-util", "macros", "rt"] } thiserror = { workspace = true } +[dev-dependencies] +astral-tokio-tar = { workspace = true } +futures = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +tar-codec = { workspace = true } +tar-framing = { workspace = true } + [lints] workspace = true diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md new file mode 100644 index 000000000..d9b558d81 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md @@ -0,0 +1,24 @@ +# minio-go Snowball fixtures + +These request bodies are generated by +`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned +in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by +RustFS Snowball extraction. + +The decoded TAR intentionally ends immediately after the final padded member +body because minio-go flushes, rather than closes, its TAR writer. The +compatibility test permits that shape only when the authenticated request body +is complete at the exact member boundary; it does not make incomplete TAR +terminators generally valid. + +Regenerate them from this directory with Go 1.25: + +```console +cd generate +go mod download +go run . -out .. +``` + +`manifest.json` records the input objects and SHA-256 digest of each captured +request body. Review changes to the manifest and binary fixtures together when +updating minio-go. diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod new file mode 100644 index 000000000..e2fd8f481 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod @@ -0,0 +1,26 @@ +module rustfs.local/snowball-fixture + +go 1.25.0 + +require github.com/minio/minio-go/v7 v7.3.0 + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum new file mode 100644 index 000000000..0027f37cd --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum @@ -0,0 +1,59 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go new file mode 100644 index 000000000..b03052943 --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go @@ -0,0 +1,193 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const minioGoVersion = "v7.3.0" + +type fixtureManifest struct { + Generator string `json:"generator"` + MinioGo string `json:"minio_go"` + GeneratedAt string `json:"generated_at"` + Objects []fixtureObject `json:"objects"` + Archives []fixtureArchive `json:"archives"` +} + +type fixtureObject struct { + Key string `json:"key"` + Body string `json:"body"` + ModTime string `json:"mod_time"` + VersionID string `json:"version_id,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` +} + +type fixtureArchive struct { + File string `json:"file"` + Compressed bool `json:"compressed"` + Length int `json:"length"` + SHA256 string `json:"sha256"` +} + +func objects() []fixtureObject { + return []fixtureObject{ + { + Key: "alpha.txt", + Body: "alpha-body", + ModTime: "2024-01-02T03:04:05Z", + VersionID: "018cc251-f400-7c22-9e8d-8b1800000001", + Headers: map[string][]string{ + "Content-Type": {"text/plain"}, + "X-Amz-Meta-Owner": {"snowball-fixture"}, + "X-Amz-Tagging": {"project=rustfs&source=minio-go"}, + }, + }, + { + Key: "nested/世界.txt", + Body: "bravo-body", + ModTime: "2024-01-02T03:05:05Z", + Headers: map[string][]string{ + "Content-Language": {"zh-CN"}, + "X-Amz-Meta-Note": {"unicode-path"}, + }, + }, + } +} + +func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) { + body := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + payload, err := io.ReadAll(request.Body) + if err != nil { + http.Error(writer, err.Error(), http.StatusInternalServerError) + return + } + body <- payload + writer.Header().Set("ETag", `"snowball-fixture"`) + writer.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{ + // The S3 authentication layer removes AWS streaming-signature framing + // before Snowball extraction sees the request body. Anonymous signing + // captures those decoded archive bytes directly. + Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous), + Secure: false, + Region: "us-east-1", + }) + if err != nil { + return nil, fmt.Errorf("construct minio client: %w", err) + } + + input := make(chan minio.SnowballObject, len(specs)) + for _, spec := range specs { + modTime, err := time.Parse(time.RFC3339, spec.ModTime) + if err != nil { + return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err) + } + headers := make(http.Header, len(spec.Headers)) + for name, values := range spec.Headers { + headers[name] = append([]string(nil), values...) + } + input <- minio.SnowballObject{ + Key: spec.Key, + Size: int64(len(spec.Body)), + ModTime: modTime, + Content: bytes.NewReader([]byte(spec.Body)), + VersionID: spec.VersionID, + Headers: headers, + } + } + close(input) + + err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{ + Opts: minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }, + InMemory: true, + Compress: compressed, + }, input) + if err != nil { + return nil, fmt.Errorf("generate snowball request: %w", err) + } + return <-body, nil +} + +func main() { + outDir := flag.String("out", "..", "fixture output directory") + flag.Parse() + + specs := objects() + archives := make([]fixtureArchive, 0, 2) + for _, fixture := range []struct { + name string + compressed bool + }{ + {name: "snowball.tar"}, + {name: "snowball.tar.s2", compressed: true}, + } { + payload, err := captureSnowball(fixture.compressed, specs) + if err != nil { + panic(err) + } + path := filepath.Join(*outDir, fixture.name) + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } + digest := sha256.Sum256(payload) + archives = append(archives, fixtureArchive{ + File: fixture.name, + Compressed: fixture.compressed, + Length: len(payload), + SHA256: hex.EncodeToString(digest[:]), + }) + } + + manifest := fixtureManifest{ + Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + MinioGo: minioGoVersion, + GeneratedAt: "2026-09-05T00:00:00Z", + Objects: specs, + Archives: archives, + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + panic(err) + } + payload = append(payload, '\n') + path := filepath.Join(*outDir, "manifest.json") + if err := os.WriteFile(path, payload, 0o644); err != nil { + panic(fmt.Errorf("write %s: %w", path, err)) + } +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json new file mode 100644 index 000000000..85191f09c --- /dev/null +++ b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json @@ -0,0 +1,51 @@ +{ + "generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball", + "minio_go": "v7.3.0", + "generated_at": "2026-09-05T00:00:00Z", + "objects": [ + { + "key": "alpha.txt", + "body": "alpha-body", + "mod_time": "2024-01-02T03:04:05Z", + "version_id": "018cc251-f400-7c22-9e8d-8b1800000001", + "headers": { + "Content-Type": [ + "text/plain" + ], + "X-Amz-Meta-Owner": [ + "snowball-fixture" + ], + "X-Amz-Tagging": [ + "project=rustfs\u0026source=minio-go" + ] + } + }, + { + "key": "nested/世界.txt", + "body": "bravo-body", + "mod_time": "2024-01-02T03:05:05Z", + "headers": { + "Content-Language": [ + "zh-CN" + ], + "X-Amz-Meta-Note": [ + "unicode-path" + ] + } + } + ], + "archives": [ + { + "file": "snowball.tar", + "compressed": false, + "length": 4096, + "sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e" + }, + { + "file": "snowball.tar.s2", + "compressed": true, + "length": 528, + "sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992" + } + ] +} diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar new file mode 100644 index 000000000..ea427f547 Binary files /dev/null and b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar differ diff --git a/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 new file mode 100644 index 000000000..b9337a766 Binary files /dev/null and b/crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2 differ diff --git a/crates/zip/tests/snowball_tar_codec_compat.rs b/crates/zip/tests/snowball_tar_codec_compat.rs new file mode 100644 index 000000000..53e37afa8 --- /dev/null +++ b/crates/zip/tests/snowball_tar_codec_compat.rs @@ -0,0 +1,548 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::io::Cursor; + +use futures::StreamExt; +use rustfs_zip::CompressionFormat; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive}; +use tar_framing::{ + FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind, + logical::{MemberExtensions, PaxState, TarReader}, +}; +use tokio::io::AsyncReadExt; + +const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0"; +const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar"); +const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2"); +const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json"); + +#[derive(Debug, Deserialize)] +struct FixtureManifest { + generator: String, + minio_go: String, + generated_at: String, + objects: Vec, + archives: Vec, +} + +#[derive(Debug, Deserialize)] +struct FixtureObject { + key: String, + body: String, + mod_time: String, + #[serde(default)] + version_id: String, + #[serde(default)] + headers: BTreeMap>, +} + +#[derive(Debug, Deserialize)] +struct FixtureArchive { + file: String, + compressed: bool, + length: usize, + sha256: String, +} + +#[derive(Debug, Eq, PartialEq)] +struct ParsedMember { + path: String, + size: u64, + mtime: Option, + body: Vec, + minio_pax: BTreeMap>>, +} + +fn sha256_hex(bytes: &[u8]) -> String { + let mut encoded = String::with_capacity(64); + for byte in Sha256::digest(bytes) { + write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail"); + } + encoded +} + +async fn decode_s2(bytes: &[u8]) -> Vec { + let mut decoder = CompressionFormat::S2 + .get_decoder(Cursor::new(bytes.to_vec())) + .expect("S2 fixture decoder should be available"); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode"); + decoded +} + +async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec { + let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec())); + let mut entries = archive.entries().expect("tokio-tar should create an entry stream"); + let mut parsed = Vec::new(); + + while let Some(entry) = entries.next().await { + let mut entry = entry.expect("tokio-tar should parse the fixture member"); + let kind = entry.header().entry_type(); + if kind == tokio_tar::EntryType::XGlobalHeader { + continue; + } + + let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path"); + let path = std::str::from_utf8(path_bytes.as_ref()) + .expect("fixture paths should be UTF-8") + .to_owned(); + let size = entry.effective_size(); + let mtime = entry.header().mtime().ok(); + let mut minio_pax = BTreeMap::new(); + if let Some(extensions) = entry + .pax_extensions() + .await + .expect("tokio-tar should parse local PAX records") + { + for extension in extensions { + let extension = extension.expect("fixture PAX record should be valid"); + let key = extension.key().expect("fixture PAX keys should be UTF-8"); + if key.starts_with("minio.") { + minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec())); + } + } + } + let mut body = Vec::new(); + entry + .read_to_end(&mut body) + .await + .expect("tokio-tar should read the fixture body"); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } + parsed +} + +fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec) -> BTreeMap>> { + for extension in state.extensions() { + for record in extension.records() { + let keyword = record.keyword(); + if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio") + && !known_keywords.contains(&keyword) + { + known_keywords.push(keyword); + } + } + } + + known_keywords + .iter() + .filter_map(|keyword| { + let record = state.effective_record(keyword)?; + let PaxRecord::Vendor { vendor, name, value } = record else { + return None; + }; + let key = format!("{vendor}.{name}"); + let value = match value { + PaxValue::Value(value) => Some(value.to_vec()), + PaxValue::Deleted => None, + }; + Some((key, value)) + }) + .collect() +} + +fn effective_mtime(header_mtime: Option, extensions: &MemberExtensions<'_>) -> Option { + let MemberExtensions::Pax(state) = extensions else { + return header_mtime; + }; + match state.effective_record(&PaxKeyword::Mtime) { + Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value), + Some(PaxRecord::Mtime(PaxValue::Deleted)) => None, + _ => header_mtime, + } +} + +fn padded_member_end(position: u64, size: u64) -> u64 { + let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512; + position + .checked_add(512) + .and_then(|position| position.checked_add(padded_size)) + .expect("fixture member end should not overflow") +} + +fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option, request_body_complete: bool) -> bool { + // The production gate must source `request_body_complete` from RustFS's + // length, checksum, and trailing-header validation state. + request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position) +} + +fn candidate_snowball_decode_policy() -> DecodePolicy { + DecodePolicy::default() + .allow_gnu(true) + .allow_all_nul_numeric_fields(true) + .max_gnu_extension_size(1_048_576) + .pax_policy( + PaxDecodePolicy::default() + .max_extension_size(1_048_576) + .max_global_extensions_size(67_108_864) + .allow_global_pax_extensions(false) + .allow_non_utf8_pax_vendor_values(false) + .allow_duplicate_pax_records(false) + .allow_global_pax_member_metadata(false) + .vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])), + ) +} + +async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec, Option, Option) { + let policy = StreamPolicy::default() + .max_pax_extension_size(1024 * 1024) + .max_global_pax_extensions_size(4 * 1024 * 1024) + .max_gnu_extension_size(128 * 1024); + let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy); + let mut parsed = Vec::new(); + let mut known_minio_keywords = Vec::new(); + let mut last_member_end = None; + + loop { + let mut frame = match reader.next_frame().await { + Ok(Some(frame)) => frame, + Ok(None) => return (parsed, None, last_member_end), + Err(error) => return (parsed, Some(error), last_member_end), + }; + assert_eq!(frame.header.kind, UstarKind::Regular); + let path = String::from_utf8( + frame + .effective_path() + .expect("tar-framing should resolve the fixture path") + .into_owned(), + ) + .expect("fixture paths should be UTF-8"); + let size = frame.header.effective_size; + let mtime = effective_mtime(frame.header.mtime, &frame.extensions); + let minio_pax = match &frame.extensions { + MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords), + MemberExtensions::Gnu { .. } => BTreeMap::new(), + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while frame + .payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-framing should read the fixture body") + { + body.extend_from_slice(&chunk); + } + last_member_end = Some(padded_member_end(frame.header.position, size)); + parsed.push(ParsedMember { + path, + size, + mtime, + body, + minio_pax, + }); + } +} + +#[test] +fn checked_in_fixtures_match_the_minio_go_manifest() { + let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON"); + assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball"); + assert_eq!(manifest.minio_go, "v7.3.0"); + assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z"); + assert_eq!(manifest.objects.len(), 2); + assert_eq!(manifest.objects[0].key, "alpha.txt"); + assert_eq!(manifest.objects[0].body, "alpha-body"); + assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z"); + assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001"); + assert_eq!( + manifest.objects[0].headers.get("X-Amz-Meta-Owner"), + Some(&vec!["snowball-fixture".to_owned()]) + ); + + for archive in &manifest.archives { + let bytes = match archive.file.as_str() { + "snowball.tar" => RAW_FIXTURE, + "snowball.tar.s2" => S2_FIXTURE, + file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"), + }; + assert_eq!(bytes.len(), archive.length); + assert_eq!(sha256_hex(bytes), archive.sha256); + assert_eq!(archive.compressed, archive.file.ends_with(".s2")); + } +} + +#[tokio::test] +async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() { + assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE); + assert_eq!(RAW_FIXTURE.len() % 512, 0); + assert!(RAW_FIXTURE.len() >= 1024); + assert!( + !RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0), + "minio-go Flush output should not contain the standard two-block terminator" + ); +} + +#[tokio::test] +async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() { + let expected = parse_with_tokio_tar(RAW_FIXTURE).await; + let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("footerless minio-go fixture should fail strict termination"); + + assert_eq!(actual, expected); + assert_eq!( + actual, + [ + ParsedMember { + path: "alpha.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_645), + body: b"alpha-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),), + ("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),), + ( + "minio.metadata.X-Amz-Tagging".to_owned(), + Some(b"project=rustfs&source=minio-go".to_vec()), + ), + ("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),), + ]), + }, + ParsedMember { + path: "nested/世界.txt".to_owned(), + size: 10, + mtime: Some(1_704_164_705), + body: b"bravo-body".to_vec(), + minio_pax: BTreeMap::from([ + ("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),), + ("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),), + ]), + }, + ] + ); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + error.position, + u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64") + ); + assert_eq!(last_member_end, Some(error.position)); +} + +#[tokio::test] +async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() { + let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await; + let error = error.expect("the real fixture should be footerless"); + assert!(is_authenticated_footerless_end(&error, last_member_end, true)); + assert!(!is_authenticated_footerless_end(&error, last_member_end, false)); + + let mut one_zero_block = RAW_FIXTURE.to_vec(); + one_zero_block.extend([0; 512]); + let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await; + let error = error.expect("one zero block is not a valid TAR terminator"); + assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker)); + assert_eq!( + last_member_end, + Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")) + ); + assert_eq!( + error.position, + u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64") + ); + assert!(!is_authenticated_footerless_end(&error, last_member_end, true)); +} + +#[tokio::test] +async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() { + let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await { + Err(error) => error, + Ok(_) => panic!("the default policy should reject minio vendor records"), + }; + assert!(default_error.to_string().contains("pax vendor extension minio.")); + + let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())) + .with_policy(candidate_snowball_decode_policy()) + .members(); + let mut bodies = Vec::new(); + loop { + let member = match members.next().await { + Ok(Some(member)) => member, + Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"), + Err(error) => { + assert!(error.to_string().contains("missing two-block end-of-archive marker")); + break; + } + }; + let Member::File { mut payload, .. } = member else { + panic!("fixture should contain only regular files"); + }; + let mut body = Vec::new(); + let mut chunk = Vec::new(); + while payload + .next_chunk(&mut chunk, 64 * 1024) + .await + .expect("tar-codec should read the fixture body") + { + body.extend_from_slice(&chunk); + } + bodies.push(body); + } + assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]); + assert!( + members + .next() + .await + .expect("the member cursor should be fused after an error") + .is_none() + ); +} + +fn pax_record(key: &str, value: &str) -> Vec { + let payload = format!("{key}={value}\n"); + let mut len = payload.len() + 3; + loop { + let record = format!("{len} {payload}"); + if record.len() == len { + return record.into_bytes(); + } + len = record.len(); + } +} + +async fn append_pax_header( + builder: &mut tokio_tar::Builder>>, + entry_type: tokio_tar::EntryType, + records: &[(&str, &str)], +) { + let mut payload = Vec::new(); + for (key, value) in records { + payload.extend(pax_record(key, value)); + } + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(entry_type); + header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64")); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload)) + .await + .expect("PAX test header should be written"); +} + +async fn append_regular(builder: &mut tokio_tar::Builder>>, path: &str) { + let body = path.as_bytes(); + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(tokio_tar::EntryType::Regular); + header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64")); + header.set_mode(0o644); + header.set_mtime(1_704_164_645); + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(body)) + .await + .expect("ordinary test member should be written"); +} + +async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await; + append_regular(&mut builder, "member.txt").await; + builder.into_inner().await.expect("policy archive should finish").into_inner() +} + +#[tokio::test] +async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() { + let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await; + let error = match TarArchive::new(Cursor::new(unknown_vendor)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"), + }; + assert!( + error + .to_string() + .contains("pax vendor extension acme.metadata.owner is not allowed") + ); + + let duplicate = archive_with_local_pax(&[ + ("minio.metadata.x-amz-meta-owner", "first"), + ("minio.metadata.x-amz-meta-owner", "second"), + ]) + .await; + let error = match TarArchive::new(Cursor::new(duplicate)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"), + }; + assert!( + error + .to_string() + .contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner") + ); +} + +#[tokio::test] +async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header( + &mut builder, + tokio_tar::EntryType::XGlobalHeader, + &[("minio.metadata.x-amz-meta-owner", "global")], + ) + .await; + append_pax_header( + &mut builder, + tokio_tar::EntryType::XHeader, + &[("minio.metadata.x-amz-meta-owner", "local")], + ) + .await; + append_regular(&mut builder, "local.txt").await; + append_regular(&mut builder, "inherited.txt").await; + let archive = builder + .into_inner() + .await + .expect("precedence archive should finish") + .into_inner(); + + let legacy = parse_with_tokio_tar(&archive).await; + let (framing, error, _) = parse_with_tar_framing(&archive).await; + assert!(error.is_none()); + assert_eq!(legacy.len(), 2); + assert_eq!(framing.len(), 2); + + let owner_key = "minio.metadata.x-amz-meta-owner"; + assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert!(!legacy[1].minio_pax.contains_key(owner_key)); + assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec()))); + assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec()))); + + let error = match TarArchive::new(Cursor::new(archive)) + .with_policy(candidate_snowball_decode_policy()) + .members() + .next() + .await + { + Err(error) => error, + Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"), + }; + assert!(error.to_string().contains("global pax extended headers are not allowed")); +} diff --git a/deny.toml b/deny.toml index fc5d25916..cfecab82f 100644 --- a/deny.toml +++ b/deny.toml @@ -37,8 +37,8 @@ unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] allow-git = [ # Temporary tokio-tar fork pinned to the reviewed parser limits, - # cancellation safety, and error-fusing change while - # astral-sh/tokio-tar#118 awaits an upstream release. + # cancellation safety, and error-fusing change while Snowball is + # prototyped against tar-codec and Swift retains its current reader. # owner: cxymds review: 2026-10 "https://github.com/cxymds/tokio-tar.git", # Official s3s repository. Temporarily pinned to the merged generic REST diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 6472481a0..924193b6e 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -13,7 +13,7 @@ - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation. - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains. -- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. +- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. - `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources. - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive. diff --git a/docs/testing/README.md b/docs/testing/README.md index 3589d8ee1..60343b9e2 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md). +The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation. + ## Naming conventions ### Reserved test-name substrings (migration gate) diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 256b41bd4..93770b74d 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -109,3 +109,13 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches ## Change checklist Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here. + +## ECStore invariant selection + +The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact. + +The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. + +The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability. + +Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review. diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index e9bec1fe4..de717d1b9 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -54,6 +54,22 @@ Fail-closed invariants every row enforces: Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection. +### PUT completion fixtures + +`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking. + +The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup. + +| Fixture | Completion boundary | +|---|---| +| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. | +| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. | +| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. | +| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. | +| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. | + +Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks. + ### Coverage gate `full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is: diff --git a/docs/testing/scanner-checkpoint-fixture.md b/docs/testing/scanner-checkpoint-fixture.md new file mode 100644 index 000000000..bc1992006 --- /dev/null +++ b/docs/testing/scanner-checkpoint-fixture.md @@ -0,0 +1,22 @@ +# Scanner Checkpoint Fixture + +The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark. + +Run the fixture and confirm the test filter selects a nonzero number of tests: + +```sh +cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture +``` + +The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well: + +```sh +RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture +``` + +A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome. + +For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store. + +The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches. diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index 02c4c7420..a97470f9a 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -252,8 +252,16 @@ pub(crate) async fn merged_list_objects_v2( .filter(|entry| merger.accepts(&entry.key().name)) .collect(); let keys: Vec = kept.iter().map(SideEntry::key).collect(); + if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) { + match fetch.side { + MergeSide::Source => { + degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?; + continue; + } + MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())), + } + } buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some)); - merger.push_page(fetch.side, keys, is_truncated, next_token); } let outcome = merger.finish(); @@ -340,10 +348,12 @@ async fn fetch_source_page( continuation_token: token, max_keys: params.max_keys, }, - // Everything under `filter.prefix` rolls into one common prefix, so a - // single bounded listing settles whether it exists. + // Everything under `filter.prefix` rolls into one common prefix. An + // empty truncated probe must still follow its cursor before declaring + // that prefix absent. SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest { prefix: Some(probe_prefix.as_str()), + continuation_token: token, max_keys: 1, ..Default::default() }, @@ -368,8 +378,8 @@ async fn fetch_source_page( } else { Vec::new() }, - false, - None, + !exists && page.is_truncated, + if exists { None } else { page.next_continuation_token }, )) } _ => { @@ -417,6 +427,17 @@ async fn local_delete_markers(store: &Arc, bucket: &str, keys: &[String #[cfg(test)] mod tests { use super::*; + use crate::app::bucket_usecase::DefaultBucketUsecase; + use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore}; + use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{ + FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig, + }; + use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response}; + use crate::app::storage_api::test::StoragePutObjReader; + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::app::storage_api::test::contract::object::ObjectIO as _; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn token(local: Option<&str>, local_done: bool) -> ListThroughToken { ListThroughToken { @@ -526,4 +547,332 @@ mod tests { assert!(degraded); assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local)); } + + /// Serves exactly the scripted S3 pages and joins every connection before + /// returning. A source retry or unexpected operation fails the test. + async fn scripted_list_source(pages: Vec) -> (String, tokio_util::task::AbortOnDropHandle>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind listing source"); + let address = listener.local_addr().expect("listing source address"); + let server = tokio::spawn(async move { + let mut requests = Vec::new(); + for body in pages { + let (mut stream, _) = listener.accept().await.expect("accept source listing"); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut chunk).await.expect("read signed listing request"); + assert!(count > 0, "source request must include complete headers"); + request.extend_from_slice(&chunk[..count]); + assert!(request.len() <= 32 * 1024, "listing request headers must be bounded"); + } + let first_line = String::from_utf8_lossy(&request) + .lines() + .next() + .expect("request line") + .to_string(); + // The SDK joins the bucket endpoint with the LIST operation's `/` path. + assert!( + first_line.starts_with("GET /source-bucket/?"), + "expected a path-style bucket-root LIST request, got {first_line:?}" + ); + assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}"); + requests.push(first_line); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.expect("write source page"); + stream.shutdown().await.expect("finish source response"); + } + requests + }); + (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server)) + } + + fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String { + let next = next + .map(|token| format!("{token}")) + .unwrap_or_default(); + let contents = key + .map(|key| format!("{key}1")) + .unwrap_or_default(); + format!( + "{truncated}{next}{contents}" + ) + } + + struct ListThroughTestState { + bucket: String, + module_enabled: bool, + } + + impl Drop for ListThroughTestState { + fn drop(&mut self) { + let sys = OnDemandMigrationSys::get(); + sys.remove(&self.bucket); + sys.set_module_enabled(self.module_enabled); + } + } + + async fn source_policy_request( + pages: Vec, + policy: SourceErrorPolicy, + resume_source: Option<&str>, + filter_prefix: Option<&str>, + ) -> (S3Result>, Vec) { + let store = shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create list-through bucket"); + store + .put_object( + &bucket, + "z-local", + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed real local listing"); + let (endpoint, server) = scripted_list_source(pages).await; + let sys = OnDemandMigrationSys::get(); + let _state_guard = ListThroughTestState { + bucket: bucket.clone(), + module_enabled: sys.is_module_enabled(), + }; + sys.set_module_enabled(true); + let config = OnDemandMigrationConfig { + version: 1, + enabled: true, + source: SourceConfig { + provider: Provider::Minio, + endpoint: Some(endpoint), + region: "us-east-1".into(), + bucket: "source-bucket".into(), + path_style: PathStyle::Path, + credentials: Some(SourceCredentials { + access_key: "test-access".into(), + secret_key: "test-secret".into(), + session_token: None, + }), + tls: TlsConfig::default(), + }, + filter: FilterConfig { + prefix: filter_prefix.map(str::to_string), + ..Default::default() + }, + policy: PolicyConfig { + list_through: true, + source_error: policy, + ..Default::default() + }, + }; + sys.apply(&bucket, Some(&config)).await; + assert!( + sys.state(&bucket).expect("ODM state installed").client().is_ok(), + "fake source client must build" + ); + let continuation_token = resume_source.map(|source| { + let token = ListThroughToken { + t: "odm-list".into(), + v: 1, + local: None, + local_done: false, + source: Some(source.into()), + source_done: false, + last_key: None, + }; + base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()) + }); + let input = ListObjectsV2Input { + bucket, + max_keys: Some(2), + continuation_token, + delimiter: filter_prefix.map(|_| "/".to_string()), + encoding_type: None, + expected_bucket_owner: None, + fetch_owner: None, + optional_object_attributes: None, + prefix: None, + request_payer: None, + start_after: None, + }; + let request = S3Request { + input, + method: http::Method::GET, + uri: http::Uri::from_static("/?list-type=2"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + let result = tokio::time::timeout( + Duration::from_secs(10), + DefaultBucketUsecase::from_global().execute_list_objects_v2(request), + ) + .await + .expect("listing must complete within its bounded source budget"); + let requests = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("source connections must finish") + .expect("source server must not panic"); + (result, requests) + } + + #[test] + #[serial_test::serial] + fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() { + run_large_stack_test("list-through-source-policy", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] { + for next in [None, Some(""), Some("stuck")] { + for key in [None, Some("a-source")] { + let (result, requests) = + source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await; + assert_eq!(requests.len(), 1, "a malformed source page must not be retried"); + assert!(requests[0].contains("continuation-token=stuck")); + assert_source_policy_result(result, policy); + } + } + let (result, requests) = source_policy_request( + vec![ + source_xml(Some("stuck"), true, Some("a-source")), + source_xml(Some("stuck"), true, None), + ], + policy, + None, + None, + ) + .await; + assert_eq!(requests.len(), 2, "the failure must occur during a real refill"); + assert!(!requests[0].contains("continuation-token=")); + assert!(requests[1].contains("continuation-token=stuck")); + assert_source_policy_result(result, policy); + } + }, + ) + .await; + }); + } + + #[test] + #[serial_test::serial] + fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() { + run_large_stack_test("list-through-empty-source-pages", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + for filter_prefix in [None, Some("photos/2024/")] { + let source_key = if filter_prefix.is_some() { + "photos/2024/a-source" + } else { + "a-source" + }; + let (result, requests) = source_policy_request( + vec![ + source_xml(Some("opaque-next"), true, None), + source_xml(None, false, Some(source_key)), + ], + SourceErrorPolicy::Propagate, + None, + filter_prefix, + ) + .await; + assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor"); + assert!(requests[1].contains("continuation-token=opaque-next")); + let response = result.expect("empty progressing source page is valid"); + assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list")); + let output = response.output; + let objects: Vec<_> = output + .contents + .unwrap_or_default() + .into_iter() + .map(|object| object.key.expect("listed object key")) + .collect(); + if filter_prefix.is_some() { + assert_eq!(objects, vec!["z-local"]); + assert_eq!( + output + .common_prefixes + .unwrap_or_default() + .into_iter() + .map(|prefix| prefix.prefix.expect("rolled-up prefix")) + .collect::>(), + vec!["photos/"] + ); + } else { + assert_eq!(objects, vec!["a-source", "z-local"]); + assert!(output.common_prefixes.unwrap_or_default().is_empty()); + } + assert_eq!(output.key_count, Some(2)); + assert_eq!(output.is_truncated, Some(false)); + assert!(output.next_continuation_token.is_none()); + } + }, + ) + .await; + }); + } + + fn assert_source_policy_result(result: S3Result>, policy: SourceErrorPolicy) { + match policy { + SourceErrorPolicy::Propagate => { + let error = result.expect_err("propagate must expose malformed pagination"); + assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY)); + assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into())); + assert_eq!(error.message(), Some("invalid_pagination")); + } + SourceErrorPolicy::NotFound => { + let response = result.expect("not_found must preserve the local listing"); + assert_eq!( + response + .headers + .get("x-rustfs-on-demand-migration-list") + .expect("local_only header"), + "local_only" + ); + let output = response.output; + assert_eq!( + output + .contents + .unwrap_or_default() + .into_iter() + .map(|object| object.key.expect("local key")) + .collect::>(), + vec!["z-local"] + ); + assert_eq!(output.is_truncated, Some(false)); + assert_eq!(output.key_count, Some(1)); + assert!(output.next_continuation_token.is_none()); + } + } + } } diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index fb9496c13..13393f797 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -394,7 +394,7 @@ impl DefaultObjectUsecase { // Bucket metadata uses the bucket name as its namespace-lock key. Load // every copy-time bucket snapshot before a same-object key can collide // with that key (for example, copying `bucket/bucket` onto itself). - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?; let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; if cp_src_dst_same && key == bucket { dst_opts.object_lock_config_snapshot = @@ -1388,4 +1388,48 @@ mod tests { .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); } + + #[tokio::test] + #[serial_test::serial] + async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("copy-sse-unreadable-{}", Uuid::new_v4()); + let source = "source.bin"; + let destination = "destination.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption copy bucket must be created"); + let mut reader = PutObjReader::from_vec(b"copied while the bucket still had a readable configuration".to_vec()); + store + .put_object(&bucket, source, &mut reader, &ObjectOptions::default()) + .await + .expect("copy source object must be written"); + install_unreadable_bucket_sse_config(&bucket).await; + + let input = CopyObjectInput::builder() + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: source.into(), + version_id: None, + }) + .bucket(bucket.clone()) + .key(destination.to_string()) + .build() + .expect("copy input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT))) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the copy"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, destination, &ObjectOptions::default()) + .await + .expect_err("a refused copy must not leave a destination object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } } diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index e5b7fb4da..2db042fa3 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -2037,7 +2037,7 @@ impl DefaultObjectUsecase { let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?; let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( bucket_sse_config.as_ref().map(|(config, _timestamp)| config), original_sse, diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index d108b1885..86d6e255a 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1485,8 +1485,9 @@ impl DefaultObjectUsecase { }; let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now); - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let bucket_sse_config = load_bucket_default_sse_config(&bucket).await; rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start); + let bucket_sse_config = bucket_sse_config?; debug!( target: "rustfs::app::object_usecase", component = "app", @@ -3912,4 +3913,121 @@ mod tests { .expect_err("writes after the zero-byte quota update must be denied"); assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 })); } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("put-sse-unreadable-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption PUT bucket must be created"); + install_unreadable_bucket_sse_config(&bucket).await; + + let payload = Bytes::from_static(b"an operator mandated encryption for this bucket"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("PUT input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT))) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the write"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("a refused PUT must not leave an object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_still_writes_plaintext_without_bucket_encryption() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("put-sse-absent-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("plaintext PUT bucket must be created"); + + let payload = Bytes::from_static(b"no default encryption is configured for this bucket"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("PUT input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT))) + .await + .expect("a bucket without default encryption must still accept a plaintext write"); + + let stored = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("the plaintext object must be readable"); + assert_eq!(stored.size, i64::try_from(payload.len()).expect("test payload length must fit i64")); + assert!( + !stored + .user_defined + .keys() + .any(|key| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION) + || key.starts_with("x-rustfs-encryption-") + || key.starts_with("x-minio-encryption-")), + "the object must carry no encryption metadata: {:?}", + stored.user_defined + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn execute_put_object_extract_refuses_a_bucket_whose_encryption_config_is_unreadable() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("extract-sse-unreadable-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("unreadable-encryption extract bucket must be created"); + install_unreadable_bucket_sse_config(&bucket).await; + + let payload = Bytes::from_static(b"archive bytes that must never be unpacked in plaintext"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("archive.tar".to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("extract PUT input must build"); + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + + let err = Box::pin(usecase.execute_put_object(&FS::new(), req)) + .await + .expect_err("an unreadable bucket encryption configuration must refuse the extract upload"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + let lookup_err = store + .get_object_info(&bucket, "archive.tar", &ObjectOptions::default()) + .await + .expect_err("a refused extract upload must not leave an object behind"); + assert!(is_err_object_not_found(&lookup_err), "{lookup_err}"); + } } diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index bb382ec11..c35619edb 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -269,6 +269,129 @@ pub(super) fn resolve_bucket_default_sse( (effective_sse, effective_kms_key_id) } +/// The bucket's default encryption configuration for a write path. +/// +/// `Ok(None)` carries one meaning only — this bucket has no default encryption +/// — and the write proceeds in plaintext exactly as before. Every other +/// outcome refuses the write rather than collapsing onto that same value: an +/// encryption blob that exists but cannot be read fails closed in +/// `get_sse_config` since rustfs/rustfs#7172, and swallowing that error here +/// stores plaintext into a bucket whose operator mandated encryption, with +/// nothing returned to the client and nothing in the object to tell it apart +/// afterwards (rustfs/backlog#2287). +/// +/// The states the lookup can report, and what each one does: +/// +/// * configured and readable — apply the bucket default; +/// * no encryption blob at all, including a bucket that does not exist and a +/// bucket whose metadata document is absent — `ConfigNotFound`, so a cold +/// cache and a missing bucket are never turned into a refusal, and the write +/// still fails later with its own `NoSuchBucket`; +/// * blob present but unparseable — deterministic, so retrying cannot help; +/// surfaces as `InternalError` until an operator repairs or removes it; +/// * the metadata read itself failed (namespace lock, quorum, disk, an +/// uninitialized metadata system) — transient, and the typed error maps to +/// the retryable `ServiceUnavailable`. +/// +/// The last two are distinguished by the typed error the accessor returns, not +/// re-derived here: [`ApiError`] already separates them. This mirrors +/// `prepare_sse_configuration` in `storage::sse`, the resolver the multipart +/// writer uses, which has always failed closed on the same lookup. +pub(super) async fn load_bucket_default_sse_config( + bucket: &str, +) -> S3Result> { + classify_bucket_default_sse_lookup(bucket, metadata_sys::get_sse_config(bucket).await) +} + +fn classify_bucket_default_sse_lookup( + bucket: &str, + lookup: Result<(ServerSideEncryptionConfiguration, OffsetDateTime), StorageError>, +) -> S3Result> { + match lookup { + Ok(config) => Ok(Some(config)), + Err(err) if err == StorageError::ConfigNotFound => Ok(None), + Err(err) => { + let api_error = ApiError::from(err); + error!( + event = "bucket_sse_config_lookup_failed", + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + result = "write_refused", + bucket = %bucket, + code = %api_error.code.as_str(), + error = %api_error, + "Bucket default encryption is unreadable; refusing the write instead of storing plaintext" + ); + Err(api_error.into()) + } + } +} + +#[cfg(test)] +mod bucket_default_sse_lookup_tests { + use super::*; + use s3s::dto::{ServerSideEncryptionByDefault, ServerSideEncryptionRule}; + use time::OffsetDateTime; + + fn sse_config() -> ServerSideEncryptionConfiguration { + ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AES256), + kms_master_key_id: None, + }), + blocked_encryption_types: None, + bucket_key_enabled: None, + }], + } + } + + #[test] + fn an_absent_configuration_still_writes_plaintext() { + let resolved = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ConfigNotFound)) + .expect("a bucket without default encryption must keep writing plaintext"); + + assert!(resolved.is_none()); + assert_eq!(resolve_bucket_default_sse(None, None, None, false), (None, None)); + } + + #[test] + fn a_readable_configuration_is_returned() { + let resolved = classify_bucket_default_sse_lookup("bucket", Ok((sse_config(), OffsetDateTime::UNIX_EPOCH))) + .expect("a readable configuration must not refuse the write") + .expect("a readable configuration must be applied"); + + assert_eq!(resolved.0.rules.len(), 1); + } + + #[test] + fn an_unreadable_configuration_refuses_the_write() { + let err = classify_bucket_default_sse_lookup( + "bucket", + Err(StorageError::other("persisted bucket encryption configuration is invalid")), + ) + .expect_err("a corrupt encryption blob must never degrade to plaintext"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + } + + #[test] + fn an_unavailable_metadata_read_refuses_the_write_as_retryable() { + let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum)) + .expect_err("an unreadable metadata subsystem must never degrade to plaintext"); + + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + } + + #[test] + fn a_missing_bucket_keeps_its_own_error() { + let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::BucketNotFound("bucket".to_string()))) + .expect_err("a bucket-not-found lookup must not be reported as an encryption failure"); + + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + } +} + #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; diff --git a/rustfs/src/app/object/test_support.rs b/rustfs/src/app/object/test_support.rs index 61b368cf6..55ebb28b2 100644 --- a/rustfs/src/app/object/test_support.rs +++ b/rustfs/src/app/object/test_support.rs @@ -96,3 +96,36 @@ pub(super) fn real_cold_fill_plan( }; plan } + +/// A store with an ambient `AppContext`, for tests that drive a handler end to +/// end without the object-data-cache overrides of +/// [`real_cold_fill_test_context`]. +pub(super) async fn real_store_test_context() -> (Arc, Arc) { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let ambient = current_app_context().expect("real-store tests require an ambient AppContext"); + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + (store, context) +} + +/// Leave the bucket in the state a damaged encryption blob produces: the raw +/// document is retained and the typed configuration stays `None`, which is the +/// durable "exists but cannot be read" signal `get_sse_config` fails closed on +/// (rustfs/rustfs#7172). +pub(super) async fn install_unreadable_bucket_sse_config(bucket: &str) { + use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata}; + + let sys = get_global_bucket_metadata_sys().expect("bucket metadata system must be initialized"); + let metadata = { + let sys = sys.read().await; + sys.get(bucket).await.expect("bucket metadata must be cached") + }; + let mut metadata = (*metadata).clone(); + metadata.encryption_config_xml = b"truncated".to_vec(); + metadata.sse_config = None; + set_bucket_metadata(bucket.to_string(), metadata) + .await + .expect("unreadable bucket encryption configuration must be installed"); +} diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 19548aa66..9f543ac95 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -29,11 +29,13 @@ pub(crate) fn EndpointServerPools( pub(crate) mod s3 { #[cfg(test)] pub(crate) use s3s::dto::{ - BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration, - ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault, - ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, + BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input, + ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, + ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration, }; pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result}; + #[cfg(test)] + pub(crate) use s3s::{S3Request, S3Response}; } pub(crate) mod admin { diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 1df46c234..8b1cf1246 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -764,8 +764,53 @@ def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: return [] +def core_requirements(root: Path) -> dict: + data = json.loads((root / ".config/ecstore-required-tests.json").read_text()) + if not data["tests"] or not data["fixtures"]: + raise ValueError("core test and fixture requirements must not be empty") + identities = [(test["suite"], test["name"]) for test in data["tests"]] + if len(set(identities)) != len(identities): + raise ValueError("duplicate core test requirement") + return data + + +def check_core_fixtures(root: Path) -> list[str]: + try: + fixtures = core_requirements(root)["fixtures"] + errors = [] + for fixture in fixtures: + path = (root / fixture["path"]).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("core fixture path escapes repository") + if not path.is_file(): + errors.append(f"{fixture['path']}: required core fixture missing") + elif hashlib.sha256(path.read_bytes()).hexdigest() != fixture["sha256"]: + errors.append(f"{fixture['path']}: core fixture sha256 mismatch") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot validate core fixtures: {error}"] + + +def check_core_listing(root: Path, listing: Path) -> list[str]: + """Check the existing CI run's selection, not a second filtered test run.""" + try: + required = core_requirements(root)["tests"] + suites = json.loads(listing.read_text())["rust-suites"] + if not isinstance(suites, dict): + raise ValueError("rust-suites must be an object") + errors = check_core_fixtures(root) + for test in required: + testcase = suites.get(test["suite"], {}).get("testcases", {}).get(test["name"], {}) + if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches": + errors.append(f"{test['invariant']}: required test not selected: {test['suite']}::{test['name']}") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot read core nextest listing: {error}"] + + def validate(root: Path) -> list[str]: errors: list[str] = [] + errors.extend(check_core_fixtures(root)) errors.extend(check_e2e_modules(root)) errors.extend(check_vault_test_groups(root)) errors.extend(check_ilm_build_budget(root)) @@ -779,6 +824,32 @@ def validate(root: Path) -> list[str]: class SelfTests(unittest.TestCase): + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / ".config").mkdir() + fixture = root / "fixture.hex" + fixture.write_text("4142") + requirements = { + "tests": [{"invariant": "commit", "suite": "store", "name": "commit_test"}], + "fixtures": [{"path": "fixture.hex", "sha256": hashlib.sha256(fixture.read_bytes()).hexdigest()}], + } + (root / ".config/ecstore-required-tests.json").write_text(json.dumps(requirements)) + listing = root / "listing.json" + good = {"ignored": False, "filter-match": {"status": "matches"}} + for case, testcase in (("selected", good), ("missing", {}), ("ignored", dict(good, ignored=True)), + ("filtered", dict(good, **{"filter-match": {"status": "mismatch"}}))): + with self.subTest(case=case): + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": testcase}}}})) + self.assertEqual(bool(check_core_listing(root, listing)), case != "selected") + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": good}}}})) + fixture.write_text("4143") + self.assertIn("sha256 mismatch", check_core_listing(root, listing)[0]) + fixture.unlink() + self.assertIn("fixture missing", check_core_listing(root, listing)[0]) + listing.write_text("not json") + self.assertIn("cannot read", check_core_listing(root, listing)[0]) + def test_ilm_lane_keeps_the_measured_cargo_build_budget(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -981,6 +1052,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_e2e_modules", return_value=[]), mock.patch(__name__ + ".check_vault_test_groups", return_value=[]), mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), + mock.patch(__name__ + ".check_core_fixtures", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_workflow_readiness", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), @@ -1393,6 +1465,11 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + if len(sys.argv) == 3 and sys.argv[1] == "--check-core": + errors = check_core_listing(ROOT, Path(sys.argv[2])) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 if len(sys.argv) == 4 and sys.argv[1] == "--check-profile": errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3])) if errors: @@ -1410,7 +1487,7 @@ def main() -> int: return 0 if sys.argv[1:]: print( - "usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING | " + "usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | " "--update-profile PROFILE LISTING PLATFORM]", file=sys.stderr, ) diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt index bf777b0f7..3bf9bd823 100644 --- a/scripts/ecstore-module-lint-register.txt +++ b/scripts/ecstore-module-lint-register.txt @@ -20,8 +20,6 @@ crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|clippy::all crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_must_use crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables -crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all -crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables crates/s3-client/src/api_error_response.rs|clippy::all crates/s3-client/src/api_error_response.rs|unused_must_use @@ -74,36 +72,18 @@ crates/ecstore/src/services/event_notification.rs|unused_variables crates/ecstore/src/services/tier/tier.rs|clippy::all crates/ecstore/src/services/tier/tier.rs|unused_must_use crates/ecstore/src/services/tier/tier.rs|unused_variables -crates/ecstore/src/services/tier/tier_admin.rs|clippy::all -crates/ecstore/src/services/tier/tier_admin.rs|unused_must_use crates/ecstore/src/services/tier/tier_admin.rs|unused_variables crates/ecstore/src/services/tier/warm_backend.rs|clippy::all crates/ecstore/src/services/tier/warm_backend.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_aliyun.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_azure.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_gcs.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_minio.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_r2.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_rustfs.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_variables crates/ecstore/src/services/tier/warm_backend_s3.rs|clippy::all crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables -crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all -crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables