mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
Compare commits
103 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d20476a66c | |||
| 9baa92563a | |||
| 21c2fb42bb | |||
| d091554ffe | |||
| c04ee41cf0 | |||
| a9691b6797 | |||
| 7db3882777 | |||
| 3f3e3f4f05 | |||
| 01e0af6312 | |||
| 3377688dab | |||
| 890ddea94b | |||
| 9f02ca6c36 | |||
| 33cd11472a | |||
| 3ff250f1cd | |||
| d795729585 | |||
| 9e6e02ea09 | |||
| 39274fc37c | |||
| 33eff4c3c4 | |||
| a2f16aa066 | |||
| 4c8b9f87e1 | |||
| 3272730c13 | |||
| 1862112d0c | |||
| cd0ac02879 | |||
| 6cf9cf7bb5 | |||
| f1f86ee9d0 | |||
| 1eef0de003 | |||
| a118d7e4fd | |||
| ed1bedf1fb | |||
| 4392f94e1a | |||
| 81d7b7d07a | |||
| e26668e62c | |||
| 8d3511c1b3 | |||
| d172d05e86 | |||
| 0d86c50760 | |||
| 526d6f667e | |||
| dcf3e4b9e8 | |||
| 04b9c8fd36 | |||
| c1f66969d7 | |||
| cfa9276fad | |||
| db8f55cb97 | |||
| 7f23a1ba91 | |||
| 1619c4be60 | |||
| 72fd7339c9 | |||
| 71e83aeec4 | |||
| 9138c24571 | |||
| e9f5318027 | |||
| 69e8ef9af5 | |||
| 0b2a46b36f | |||
| 56509ead1f | |||
| ffe889ad59 | |||
| e11ce2f132 | |||
| eca6bc1600 | |||
| 85be26b3c1 | |||
| ebbcfa3ac2 | |||
| ebd0531124 | |||
| 4421d4829f | |||
| d6c62b9601 | |||
| d91086d094 | |||
| 69719c257e | |||
| 67a19021b5 | |||
| 0ff3d4cbf4 | |||
| 6f29431a65 | |||
| 5a4c063d16 | |||
| e2be34cade | |||
| d60a77b750 | |||
| 307f50ee1b | |||
| c48a6330d0 | |||
| 8ac2ff5c61 | |||
| eb41f45175 | |||
| 122d200675 | |||
| 161e515c72 | |||
| 83cf063b45 | |||
| f8bbfcbeb1 | |||
| 710dcb4865 | |||
| f5cced910a | |||
| 8c9249054f | |||
| 7c2b513613 | |||
| 00844721ff | |||
| 068a0c2b8c | |||
| 5b54c4303d | |||
| 6178083985 | |||
| e16c07b9cd | |||
| 1ac28d6459 | |||
| 9b66040a02 | |||
| 7710f70fda | |||
| aa4d3317ed | |||
| f704d015d6 | |||
| 6b86d44cac | |||
| e3c15f012c | |||
| d2b1003612 | |||
| 36deab8670 | |||
| e11fcfbd08 | |||
| 11eecdc888 | |||
| 80eb4244a3 | |||
| e4da9bd718 | |||
| e28430ab3d | |||
| db4707f187 | |||
| 3a0dbccc2e | |||
| 846517625b | |||
| f21e88b112 | |||
| a5594c3d89 | |||
| fc927caadd | |||
| b7e6334c13 |
@@ -252,10 +252,16 @@ test-group = 'ecstore-serial-flaky'
|
||||
# cluster, so it keeps the lane's parallel-safe / no-external-dependency
|
||||
# properties. The RustFS warm backend has no loopback guard (that guard is
|
||||
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
|
||||
#
|
||||
# Disk compression (backlog#1848): the `compression` module joins the smoke
|
||||
# lane so the multipart disk-compression roundtrips (restored after
|
||||
# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate.
|
||||
# Single-node servers on random ports with isolated temp dirs — meets the
|
||||
# admission criteria unchanged.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
|
||||
@@ -182,7 +182,12 @@ jobs:
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
# Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2
|
||||
# (weekly schedule / manual dispatch only — dormant rio-v2 variant, see
|
||||
# rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md).
|
||||
# The second build below stays despite the reduced cadence: it warms the
|
||||
# rio-v2,e2e-test-hooks feature resolution the scheduled build restores,
|
||||
# which keeps that lane inside its 30-minute timeout.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
|
||||
@@ -533,7 +533,12 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary-rio-v2:
|
||||
name: Build RustFS Debug Binary (rio-v2)
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
||||
# default build, so this full-suite lane runs only on the weekly schedule
|
||||
# and manual dispatch. Per-PR cfg-seam coverage stays with
|
||||
# test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
|
||||
# docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
@@ -824,6 +829,9 @@ jobs:
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
# event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
|
||||
# with it (see the dormant-variant comment on that job).
|
||||
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
|
||||
@@ -94,6 +94,7 @@ jobs:
|
||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -118,6 +119,7 @@ jobs:
|
||||
short_sha=""
|
||||
is_prerelease=false
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
@@ -137,6 +139,7 @@ jobs:
|
||||
# Extract version info from commit message or use commit SHA
|
||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||
source_ref="$HEAD_SHA"
|
||||
|
||||
# Determine build type based on triggering workflow event and ref
|
||||
triggering_event="$TRIGGERING_EVENT"
|
||||
@@ -261,6 +264,23 @@ jobs:
|
||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||
tag_ref="refs/tags/$input_version"
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
if [[ "$input_version" == v* ]]; then
|
||||
tag_ref="refs/tags/${input_version#v}"
|
||||
else
|
||||
tag_ref="refs/tags/v$input_version"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
echo "❌ Release tag not found for Docker build: $input_version"
|
||||
exit 1
|
||||
fi
|
||||
source_ref="$tag_ref"
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
@@ -271,6 +291,7 @@ jobs:
|
||||
echo "short_sha=$short_sha"
|
||||
echo "is_prerelease=$is_prerelease"
|
||||
echo "create_latest=$create_latest"
|
||||
echo "source_ref=$source_ref"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker Build Summary:"
|
||||
@@ -281,6 +302,7 @@ jobs:
|
||||
echo " - Short SHA: $short_sha"
|
||||
echo " - Is prerelease: $is_prerelease"
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
@@ -308,6 +330,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
@@ -397,7 +420,8 @@ jobs:
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||
|
||||
+4
-1
@@ -101,7 +101,10 @@ refactors.
|
||||
|
||||
The `rustfs` binary crate composes these libraries into the running server.
|
||||
`ecstore` remains the storage engine at the architectural center; its internal
|
||||
module split is tracked under `docs/architecture/`.
|
||||
module split is tracked under `docs/architecture/`. `rio-v2` is the
|
||||
feature-gated MinIO on-disk format compatibility I/O layer; it ships in no
|
||||
default build (lifecycle:
|
||||
[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)).
|
||||
|
||||
## Architecture Invariants
|
||||
|
||||
|
||||
Generated
+79
-77
@@ -278,6 +278,7 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
||||
dependencies = [
|
||||
"bigdecimal",
|
||||
"bon",
|
||||
"crc32fast",
|
||||
"digest 0.11.3",
|
||||
"log",
|
||||
"miniz_oxide 0.9.1",
|
||||
@@ -289,9 +290,11 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"snap",
|
||||
"strum",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1162,9 +1165,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-eventstream"
|
||||
version = "0.61.1"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944"
|
||||
checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
@@ -1195,9 +1198,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.2.0"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
|
||||
checksum = "3c1c8a04cb31ba74d0115af5a890bb8c0d48fba64b52812fa13929a6ef0cc83c"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-protocol-test",
|
||||
@@ -1277,9 +1280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.12.1"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1"
|
||||
checksum = "483b858ff67522011c4786310c5cd8fd88d0be7ea3d5f1a48328446300c4269e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -1343,9 +1346,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.6.1"
|
||||
version = "1.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8"
|
||||
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -3761,7 +3764,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -5133,9 +5136,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http-body-util"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
|
||||
checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -5957,7 +5960,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=ce6338661179c8be22e516b00af7483f151485a7#ce6338661179c8be22e516b00af7483f151485a7"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
@@ -6259,9 +6262,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique"
|
||||
version = "0.1.29"
|
||||
version = "0.1.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2e394c63e2d1a30aeb3b9392ecf3439d8475d2df810a8f4f6e66d6866754017"
|
||||
checksum = "dedbf06ffeef4c37990c73636fbd993aa34fb1948afd736e6114f239220993db"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -6289,9 +6292,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-macro"
|
||||
version = "0.1.20"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "786df1fd0abebd0db685f7e9a353c78756d4b370fb98a52376c2015fa55f141f"
|
||||
checksum = "f4fb1f30185f53f7f6e4c9e46745c1a1350af8e77fda5a88aded44b0637a82e0"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"darling 0.23.0",
|
||||
@@ -6318,9 +6321,9 @@ checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9"
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer"
|
||||
version = "0.1.25"
|
||||
version = "0.1.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82cdde44d241dab7fc8b7a32e0eb5dae6cd28f8de80b59f9a1e9f2f0b05e485e"
|
||||
checksum = "20bd17c1a3ca2719e31f19ce77a853948dc2102f35976b92276c42a64fdc5f3f"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"crossbeam-queue",
|
||||
@@ -6339,9 +6342,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer-core"
|
||||
version = "0.1.19"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57379b7ee2272efaeaaa6de062503563e57333b24aadc7f2255b3d602899e8b"
|
||||
checksum = "f1a55b6aae1d85c557c729564c4e2b32a26dc65ba2d90d9647ca01f2bd4854c4"
|
||||
dependencies = [
|
||||
"derive-where",
|
||||
"itertools 0.14.0",
|
||||
@@ -6366,7 +6369,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=ce6338661179c8be22e516b00af7483f151485a7#ce6338661179c8be22e516b00af7483f151485a7"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
@@ -6882,7 +6885,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -8043,7 +8046,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -8063,7 +8066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -8084,7 +8087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8097,7 +8100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -9090,7 +9093,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9200,6 +9203,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"sha2 0.11.0",
|
||||
"shadow-rs",
|
||||
"snap",
|
||||
"socket2",
|
||||
"subtle",
|
||||
"sysinfo",
|
||||
@@ -9227,7 +9231,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"const-str",
|
||||
@@ -9250,7 +9254,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9266,7 +9270,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"hotpath",
|
||||
@@ -9284,7 +9288,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9297,7 +9301,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9307,7 +9311,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9321,7 +9325,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9342,7 +9346,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9352,7 +9356,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9491,7 +9495,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9501,7 +9505,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9528,7 +9532,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -9559,7 +9563,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9600,7 +9604,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -9613,16 +9617,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"num_cpus",
|
||||
"rustfs-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-utils",
|
||||
"sysinfo",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
@@ -9679,7 +9681,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9706,7 +9708,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9755,7 +9757,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -9778,7 +9780,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -9801,7 +9803,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9820,7 +9822,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"humantime",
|
||||
@@ -9835,7 +9837,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9870,7 +9872,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9890,7 +9892,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9907,7 +9909,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9962,7 +9964,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9993,7 +9995,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10055,7 +10057,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10079,7 +10081,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10097,7 +10099,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10135,7 +10137,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10158,7 +10160,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10166,7 +10168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10175,7 +10177,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10205,7 +10207,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10224,7 +10226,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10264,7 +10266,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10272,7 +10274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10290,7 +10292,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10305,7 +10307,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10359,7 +10361,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10375,7 +10377,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10396,7 +10398,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10433,7 +10435,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10475,7 +10477,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10561,9 +10563,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-connector"
|
||||
version = "0.23.7"
|
||||
version = "0.23.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09a5abe04eec18f8b9fbe87885bcaee6426de80bbc579958c0bc064b728ee617"
|
||||
checksum = "1babecfcc65b139b812e74bcc7f9ec7b4e00db659fd42d99567b7e77f0c714c6"
|
||||
dependencies = [
|
||||
"futures-io",
|
||||
"futures-rustls",
|
||||
@@ -11802,7 +11804,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
|
||||
+54
-54
@@ -41,7 +41,7 @@ members = [
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -154,7 +154,7 @@ hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
http-body-util = "0.1.5"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.2.0" }
|
||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.22.0"
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -231,9 +231,9 @@ aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
base64 = "0.23.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
@@ -348,8 +348,8 @@ russh-sftp = "2.4.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use base64_simd::STANDARD;
|
||||
|
||||
|
||||
@@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||
|
||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||
|
||||
@@ -476,13 +476,19 @@ impl Checksum for Xxhash64 {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
struct Md5 {
|
||||
hasher: md5::Md5,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
impl Md5 {
|
||||
fn update(&mut self, bytes: &[u8]) {
|
||||
use md5::Digest;
|
||||
|
||||
@@ -353,6 +353,11 @@ pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
|
||||
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
|
||||
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
||||
|
||||
/// Default detailed PUT stage metrics enabled
|
||||
/// Default value: false
|
||||
/// Environment variable: RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED
|
||||
pub const DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED: bool = false;
|
||||
|
||||
/// Default logs export enabled
|
||||
/// It is used to enable or disable exporting logs
|
||||
/// Default value: true
|
||||
|
||||
@@ -137,6 +137,37 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
/// Request the object-transaction fencing contract used by storage-owned
|
||||
/// cleanup receipts and lock-window optimizations.
|
||||
///
|
||||
/// This is fail-closed: enabling the writer without a live fleet proof rejects
|
||||
/// the commit rather than silently using a legacy-safe path.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_WRITE: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// object transaction fencing contract.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE);
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED);
|
||||
|
||||
/// Request preserving legacy per-part checksum metadata during data movement.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// data-movement per-part checksum sidecar.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -649,4 +680,22 @@ mod remote_version_state_tests {
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED,
|
||||
"RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_transaction_fencing_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_OBJECT_TRANSACTION_FENCING_WRITE, "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED,
|
||||
"RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,10 @@ pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENAB
|
||||
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
||||
|
||||
/// Enables detailed per-stage PUT metrics. Disabled by default because each
|
||||
/// PUT records multiple timers and histograms when attribution is active.
|
||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
@@ -141,6 +145,7 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -67,7 +67,10 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(log_path) = log_path else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
@@ -25,6 +26,15 @@ fn generate_compressible_data(size: usize) -> Vec<u8> {
|
||||
data
|
||||
}
|
||||
|
||||
/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part
|
||||
/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload
|
||||
/// Pending truncation (rustfs/rustfs#5957).
|
||||
fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
|
||||
(0..size)
|
||||
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
|
||||
let bucket_path = PathBuf::from(temp_dir).join(bucket);
|
||||
let mut part_files = Vec::new();
|
||||
@@ -55,9 +65,14 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let process = Command::new(&binary_path)
|
||||
// Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR
|
||||
// capture (survives the temp-dir cleanup on Drop and is uploaded as a CI
|
||||
// artifact); without the env var the child inherits stdio as before.
|
||||
let mut command = Command::new(&binary_path);
|
||||
command
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
@@ -66,8 +81,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
])
|
||||
.spawn()?;
|
||||
]);
|
||||
crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?;
|
||||
let process = command.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
@@ -154,3 +170,647 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MULTIPART_COMPRESSION_BUCKET: &str = "compression-multipart-bucket";
|
||||
const MPU_PART1_SIZE: usize = 5 * 1024 * 1024;
|
||||
const MPU_PART2_SIZE: usize = 1024 * 1024;
|
||||
|
||||
async fn multipart_upload(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
parts: &[&[u8]],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::with_capacity(parts.len());
|
||||
for (i, part) in parts.iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.to_vec()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_range(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
range: &str,
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let response = client.get_object().bucket(bucket).key(key).range(range).send().await?;
|
||||
Ok(response.body.collect().await?.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// Multipart disk compression roundtrip: parts are written as independent
|
||||
/// compressed streams and every GET shape must reassemble the original bytes
|
||||
/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression
|
||||
/// entirely).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)");
|
||||
|
||||
// Full GET must reassemble both independently compressed parts.
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range fully inside part 1.
|
||||
let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?;
|
||||
assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// Range fully inside part 2.
|
||||
let part2_start = MPU_PART1_SIZE + 4096;
|
||||
let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1;
|
||||
let range_inside_part2 = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={part2_start}-{part2_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_inside_part2[..],
|
||||
&original_data[part2_start..part2_end + 1],
|
||||
"part-2 range mismatch"
|
||||
);
|
||||
|
||||
// Suffix range (last 128 KiB, entirely in part 2).
|
||||
let suffix_len = 128 * 1024;
|
||||
let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
// partNumber GETs must return each original part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
info!("Multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket";
|
||||
|
||||
/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the
|
||||
/// disk-compression path runs and each part is stored as many compressed blocks — the shape that
|
||||
/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return
|
||||
/// the exact original bytes, and the stored size must show the data really was compressed.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart high-ratio binary compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-high-ratio.txt";
|
||||
let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7);
|
||||
let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
|
||||
// margin still proves the parts were stored compressed rather than raw or double-encoded.
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size as u64) / 2,
|
||||
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
|
||||
);
|
||||
info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)");
|
||||
|
||||
info!("step: full GET");
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
info!("step: boundary range GET");
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_HIGH_RATIO_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
info!("step: partNumber GET");
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch");
|
||||
|
||||
info!("Multipart high-ratio binary compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket";
|
||||
const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024;
|
||||
const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied
|
||||
/// range must be decompressed on read and re-compressed into the destination part, so the final
|
||||
/// object has to match "source prefix + uploaded tail" byte for byte.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart upload-part-copy compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
|
||||
// Source object: a plain PUT that goes through the single-stream compression path.
|
||||
let source_key = "copy-source.txt";
|
||||
let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(source_key)
|
||||
.body(ByteStream::from(source_data.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Destination object: part 1 copied from the source, part 2 uploaded directly.
|
||||
let target_key = "copy-target.txt";
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec();
|
||||
expected_data.extend_from_slice(&part2);
|
||||
let total_size = expected_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let copy_part = client
|
||||
.upload_part_copy()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}"))
|
||||
.copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1))
|
||||
.send()
|
||||
.await?;
|
||||
let copy_etag = copy_part
|
||||
.copy_part_result()
|
||||
.and_then(|r| r.e_tag())
|
||||
.ok_or("missing copy part etag")?
|
||||
.to_string();
|
||||
|
||||
let uploaded_part = client
|
||||
.upload_part()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build())
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_part.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch");
|
||||
|
||||
info!("Multipart upload-part-copy compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket";
|
||||
const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024;
|
||||
|
||||
/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one
|
||||
/// compressed part stream, and a suffix range must resolve inside the trailing part.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting three-part multipart compression partNumber test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-three-parts.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
original_data.extend_from_slice(&part3);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
|
||||
// Every partNumber GET must return exactly the bytes of the corresponding uploaded part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
// Suffix range (last 64 KiB) resolves inside the trailing part.
|
||||
let suffix_len = 64 * 1024;
|
||||
let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
info!("Three-part multipart compression partNumber test passed");
|
||||
env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
|
||||
|
||||
async fn start_rustfs_with_compression_and_sse(
|
||||
env: &mut RustFSTestEnvironment,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use base64::Engine;
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
// Server output goes to a file inside the per-test temp dir so a failing
|
||||
// run can be diagnosed from the child's logs.
|
||||
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
|
||||
let server_log_err = server_log.try_clone()?;
|
||||
let process = Command::new(&binary_path)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
|
||||
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
|
||||
.stdout(std::process::Stdio::from(server_log))
|
||||
.stderr(std::process::Stdio::from(server_log_err))
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
"--access-key",
|
||||
&env.access_key,
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
])
|
||||
.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
||||
/// shape must still return the original plaintext bytes. Physical size must shrink because the
|
||||
/// compression runs before encryption.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
|
||||
init_logging();
|
||||
info!("Starting SSE-S3 multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression_and_sse(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-sse-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::new();
|
||||
for (i, part) in [&part1, &part2].into_iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
assert_eq!(
|
||||
head_response.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::Aes256),
|
||||
"HEAD must report SSE-S3"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary must decrypt and decompress across parts.
|
||||
let boundary_start = MPU_PART1_SIZE - 64 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_SSE_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"SSE-S3 boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch");
|
||||
|
||||
info!("SSE-S3 multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end
|
||||
|
||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions.
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||
|
||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||
|
||||
@@ -30,10 +30,12 @@ use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput,
|
||||
DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput,
|
||||
GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput,
|
||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -76,6 +78,25 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
];
|
||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
];
|
||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
/// Wire prefix of the SSE-C passthrough replication transport headers
|
||||
/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a
|
||||
/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with
|
||||
/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO /
|
||||
/// generic S3, which silently discard unknown x-* headers.
|
||||
const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-";
|
||||
const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm";
|
||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||
|
||||
@@ -91,6 +112,9 @@ pub enum Operation {
|
||||
GetObject,
|
||||
HeadObject,
|
||||
DeleteObject,
|
||||
GetObjectTagging,
|
||||
PutObjectTagging,
|
||||
DeleteObjectTagging,
|
||||
ListObjectVersions,
|
||||
CreateMultipartUpload,
|
||||
UploadPart,
|
||||
@@ -118,6 +142,61 @@ pub enum FaultAction {
|
||||
WrongEtag,
|
||||
}
|
||||
|
||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||
/// sender-side tests can assert what a real target would receive.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReplicationTimestampHeaders {
|
||||
pub tagging: Option<String>,
|
||||
pub retention: Option<String>,
|
||||
pub legalhold: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicationTimestampHeaders {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-proxy related headers observed on a request, journaled so proxy
|
||||
/// tests can assert the exact wire contract: the anti-loop marker present,
|
||||
/// the replication-check exemption absent, and the client SSE-C key family
|
||||
/// forwarded verbatim. The SSE-C key value itself is never retained — only
|
||||
/// its presence.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProxyHeaderSnapshot {
|
||||
pub source_proxy_request: Option<String>,
|
||||
pub replication_check: Option<String>,
|
||||
pub ssec_algorithm: Option<String>,
|
||||
pub ssec_key_present: bool,
|
||||
pub ssec_key_md5: Option<String>,
|
||||
/// Whether the request carried any `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header, so fail-closed tests can assert the
|
||||
/// sender really shipped the material a dropping target discarded.
|
||||
pub ssec_transport_present: bool,
|
||||
}
|
||||
|
||||
impl ProxyHeaderSnapshot {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"])
|
||||
.map(bounded_journal_value),
|
||||
replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"),
|
||||
ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value),
|
||||
ssec_transport_present: headers
|
||||
.keys()
|
||||
.any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Credential-free request metadata retained for deterministic assertions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestRecord {
|
||||
@@ -131,6 +210,8 @@ pub struct RequestRecord {
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub proxy_headers: ProxyHeaderSnapshot,
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
@@ -146,6 +227,10 @@ struct ControlState {
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
assign_own_multipart_version_ids: bool,
|
||||
/// MinIO-like mode: silently discard non-whitelisted replication
|
||||
/// transport headers instead of storing them (see
|
||||
/// [`REPLICATION_SSE_TRANSPORT_PREFIX`]).
|
||||
drop_unlisted_replication_headers: bool,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
total_bytes: usize,
|
||||
@@ -167,6 +252,12 @@ struct ObjectVersion {
|
||||
delete_marker: bool,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
/// Object tags as ordered key/value pairs (PutObjectTagging replaces the
|
||||
/// whole set, DeleteObjectTagging clears it).
|
||||
tags: Vec<(String, String)>,
|
||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||
/// target behavior); empty when the drop mode discarded them.
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -176,6 +267,7 @@ struct MultipartState {
|
||||
version_id: String,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
parts: BTreeMap<i32, MultipartPart>,
|
||||
}
|
||||
|
||||
@@ -396,6 +488,15 @@ impl FakeS3Target {
|
||||
|
||||
/// Mint own version ids for the multipart path only — models a target
|
||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||
/// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header instead of storing it. The default (off)
|
||||
/// models a RustFS target, which preserves the headers and echoes SSE-C
|
||||
/// evidence (`x-amz-server-side-encryption-customer-algorithm`) on
|
||||
/// HEAD/GET of the replica.
|
||||
pub fn drop_unlisted_replication_headers(&self, enabled: bool) {
|
||||
lock(&self.backend.store).drop_unlisted_replication_headers = enabled;
|
||||
}
|
||||
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
@@ -536,7 +637,17 @@ impl S3Access for FaultAccess {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok());
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||
let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers());
|
||||
let fault = record_request(
|
||||
&self.control,
|
||||
operation,
|
||||
context.method().clone(),
|
||||
parsed,
|
||||
content_length,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
);
|
||||
if let Some(RequestFault {
|
||||
action: FaultAction::Status(status),
|
||||
..
|
||||
@@ -575,6 +686,9 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
||||
"GetObject" => Operation::GetObject,
|
||||
"HeadObject" => Operation::HeadObject,
|
||||
"DeleteObject" => Operation::DeleteObject,
|
||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||
"UploadPart" => Operation::UploadPart,
|
||||
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
||||
@@ -589,6 +703,8 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
content_length: Option<u64>,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
proxy_headers: ProxyHeaderSnapshot,
|
||||
) -> Option<RequestFault> {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
@@ -613,6 +729,8 @@ fn record_request(
|
||||
part_number: parsed.part_number,
|
||||
content_length,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
@@ -679,6 +797,15 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||
(&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::GetObjectTagging
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::PutObjectTagging
|
||||
}
|
||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::DeleteObjectTagging
|
||||
}
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
@@ -746,6 +873,29 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
||||
Ok(version_id.to_string())
|
||||
}
|
||||
|
||||
/// Capture the SSE-C passthrough transport headers a replication PUT carried.
|
||||
/// Returns an empty set in the MinIO-like drop mode.
|
||||
fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> {
|
||||
if drop_unlisted {
|
||||
return Vec::new();
|
||||
}
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX))
|
||||
.filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// SSE-C evidence a RustFS-like target echoes for a stored passthrough
|
||||
/// replica: the customer algorithm restored from the transport headers.
|
||||
fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option<String> {
|
||||
version
|
||||
.replication_sse_headers
|
||||
.iter()
|
||||
.find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER)
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
|
||||
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
||||
header_value(headers, &SOURCE_ETAG_HEADERS)
|
||||
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
||||
@@ -1093,6 +1243,33 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option<
|
||||
Ok(version.clone())
|
||||
}
|
||||
|
||||
/// Replace (or clear, with an empty vec) the tag set of the addressed
|
||||
/// version, returning its version id. Mirrors `find_version` addressing:
|
||||
/// explicit version id or the latest version, delete markers rejected.
|
||||
fn set_version_tags(
|
||||
state: &mut StoreState,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
tags: Vec<(String, String)>,
|
||||
) -> S3Result<String> {
|
||||
// Resolve first (immutable) so the error paths match find_version.
|
||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||
let versions = state
|
||||
.buckets
|
||||
.get_mut(bucket)
|
||||
.expect("bucket existence checked by find_version")
|
||||
.objects
|
||||
.get_mut(key)
|
||||
.expect("key existence checked by find_version");
|
||||
let version = versions
|
||||
.iter_mut()
|
||||
.find(|version| version.version_id == resolved)
|
||||
.expect("version existence checked by find_version");
|
||||
version.tags = tags;
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl S3 for FakeBackend {
|
||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
@@ -1189,7 +1366,10 @@ impl S3 for FakeBackend {
|
||||
let input = req.input;
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||
let (assign_own, drop_unlisted) = {
|
||||
let state = lock(&self.store);
|
||||
(state.assign_own_version_ids, state.drop_unlisted_replication_headers)
|
||||
};
|
||||
let version_id = new_version_id(&headers, assign_own)?;
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
Some(value) => value,
|
||||
@@ -1206,6 +1386,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
};
|
||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1226,6 +1408,7 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectOutput {
|
||||
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
||||
@@ -1235,6 +1418,7 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
@@ -1249,6 +1433,7 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(HeadObjectOutput {
|
||||
content_length: Some(version.body.len() as i64),
|
||||
@@ -1257,12 +1442,79 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let tag_set: TagSet = version
|
||||
.tags
|
||||
.into_iter()
|
||||
.map(|(key, value)| Tag {
|
||||
key: Some(key),
|
||||
value: Some(value),
|
||||
})
|
||||
.collect();
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectTaggingOutput {
|
||||
tag_set,
|
||||
version_id: Some(ObjectVersionId::from(version.version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let tags = input
|
||||
.tagging
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default()))
|
||||
.collect();
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(DeleteObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
@@ -1339,6 +1591,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: true,
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1366,9 +1620,10 @@ impl S3 for FakeBackend {
|
||||
ensure_upload_budget(&state)?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let upload_id = Uuid::new_v4().to_string();
|
||||
// Read the flag before the mutable borrow of `state.uploads` below
|
||||
// Read the flags before the mutable borrow of `state.uploads` below
|
||||
// (and never re-lock the store: the mutex is not reentrant).
|
||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||
let drop_unlisted = state.drop_unlisted_replication_headers;
|
||||
let version_id = new_version_id(&headers, mint_own)?;
|
||||
state.uploads.insert(
|
||||
upload_id.clone(),
|
||||
@@ -1378,6 +1633,7 @@ impl S3 for FakeBackend {
|
||||
version_id,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -1515,6 +1771,7 @@ impl S3 for FakeBackend {
|
||||
version_id: upload.version_id.clone(),
|
||||
content_type: upload.content_type.clone(),
|
||||
metadata: upload.metadata.clone(),
|
||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
selected,
|
||||
@@ -1541,6 +1798,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: upload.content_type,
|
||||
metadata: upload.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: upload.replication_sse_headers,
|
||||
};
|
||||
let mut state = lock(&self.store);
|
||||
let current = state
|
||||
@@ -1699,6 +1958,111 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("plain")
|
||||
.body(ByteStream::from_static(b"plain"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("stamped")
|
||||
.body(ByteStream::from_static(b"stamped"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let requests = target.requests();
|
||||
let plain = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||
.expect("plain PUT must be journaled");
|
||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||
|
||||
let stamped = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||
.expect("stamped PUT must be journaled");
|
||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Default mode is RustFS-like: SSE-C passthrough transport headers are
|
||||
/// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is
|
||||
/// MinIO-like: the headers are silently discarded, so no evidence comes
|
||||
/// back — the exact difference the N2 fail-closed audit keys on. Both
|
||||
/// modes journal that the sender shipped the transport headers.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
let put_with_transport_headers = |key: &'static str| {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"ciphertext"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-replication-ssec-algorithm", "AES256");
|
||||
headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA==");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
};
|
||||
|
||||
put_with_transport_headers("kept").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), Some("AES256"));
|
||||
let get = client.get_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(get.sse_customer_algorithm(), Some("AES256"));
|
||||
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
put_with_transport_headers("dropped").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("dropped").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence");
|
||||
|
||||
let requests = target.requests();
|
||||
for key in ["kept", "dropped"] {
|
||||
let record = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key))
|
||||
.expect("PUT must be journaled");
|
||||
assert!(
|
||||
record.proxy_headers.ssec_transport_present,
|
||||
"the journal must prove the sender shipped the transport headers for {key}"
|
||||
);
|
||||
}
|
||||
let plain_head = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::HeadObject)
|
||||
.expect("HEAD must be journaled");
|
||||
assert!(!plain_head.proxy_headers.ssec_transport_present);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! assert_sdk_error {
|
||||
($error:expr, $status:expr, $code:expr) => {{
|
||||
let error = &$error;
|
||||
@@ -2964,6 +3328,7 @@ mod tests {
|
||||
version_id: index.to_string(),
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
replication_sse_headers: Vec::new(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -2985,6 +3350,8 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
Some(0),
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
}
|
||||
let records = lock(&control).requests.clone();
|
||||
@@ -3006,6 +3373,8 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
None,
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
|
||||
@@ -1828,33 +1828,36 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
||||
/// Reverting the multipart compression fix must fail this test.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
|
||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-fallback";
|
||||
let bucket = "inline-multipart-compression-roundtrip";
|
||||
cluster.create_test_bucket(bucket).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
let key = "multipart/compression-disabled.txt";
|
||||
let key = "multipart/compressed.txt";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1871,6 +1874,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
||||
cluster.start().await?;
|
||||
|
||||
@@ -1890,14 +1894,21 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
ReaderPathExpectation::for_class(
|
||||
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
|
||||
LEGACY_DUPLEX,
|
||||
MULTIPART,
|
||||
COMPRESSED,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
PartNumberReaderPathExpectation::new(
|
||||
bucket,
|
||||
multipart_key,
|
||||
&second_part,
|
||||
multipart_body.len(),
|
||||
COMPRESSED,
|
||||
LEGACY_DUPLEX,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||
@@ -2353,7 +2364,11 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
hot_client.create_bucket().bucket(bucket).send().await?;
|
||||
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
|
||||
|
||||
let key = "transition/mixed-multipart.bin";
|
||||
// `.zip` sits on the disk-compression exclusion list: this test pins
|
||||
// msgpack compat controls across ILM transition, and a compressed object
|
||||
// would classify as `compressed` instead of `remote` (and the warm-tier
|
||||
// read path does not decode compression — tracked separately).
|
||||
let key = "transition/mixed-multipart.zip";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
|
||||
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
|
||||
assert!(
|
||||
|
||||
@@ -97,7 +97,7 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestRe
|
||||
|
||||
let envs = [
|
||||
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
|
||||
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "false"),
|
||||
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_ILM_PROCESS_TIME", "1"),
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
|
||||
@@ -486,7 +486,6 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
|
||||
/// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "pins rustfs/rustfs#6025: GET on a transitioned managed-SSE object silently returns corrupt bytes (fails with enforcement on AND off, so it is not an authorization regression); un-ignore with the fix"]
|
||||
async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
|
||||
@@ -2610,17 +2610,20 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
assert_eq!(payload["Status"], "OK");
|
||||
assert_eq!(payload["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["ActiveMutation"], true);
|
||||
assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(payload["Targets"][0]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
// A RustFS target adopts the source version id, so the P1-19
|
||||
// version-identity probe passes.
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||
// A RustFS target preserves the SSE-C passthrough transport headers and
|
||||
// echoes the customer algorithm on the replication-check HEAD (N2).
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["SsecPassthrough"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
let target_client = target_env.create_s3_client();
|
||||
let versions = target_client
|
||||
@@ -4649,6 +4652,410 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// N2 (backlog#1675 P1-22): SSE-C passthrough replication to a target that
|
||||
/// silently drops the `X-Rustfs-Replication-*` transport headers (MinIO-like
|
||||
/// behavior, modeled by the fake target's drop mode) used to report COMPLETED
|
||||
/// while the replica had irrecoverably lost its decryption material — the red
|
||||
/// light this test was born failing on. Fail-closed contract now under test:
|
||||
/// the first attempt PUTs, HEAD-backs the replica, finds no SSE-C evidence,
|
||||
/// records the target Unsupported and reports FAILED; a second SSE-C object
|
||||
/// fails without any PUT reaching the target (capability cache, proven from
|
||||
/// the target journal); plaintext objects still replicate COMPLETED.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "ssec-drop-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "ssec-drop-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let put_ssec = |key: &'static str| {
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"ssec fail-closed payload"))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
};
|
||||
|
||||
// First SSE-C object: the audit must catch the dropped material.
|
||||
put_ssec("ssec-first.txt").await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "ssec-first.txt", "FAILED", true).await?;
|
||||
|
||||
let requests = target.take_requests();
|
||||
let first_put = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject && record.key.as_deref() == Some("ssec-first.txt"))
|
||||
.ok_or("the first SSE-C object must have been PUT (capability was Unknown)")?;
|
||||
assert!(
|
||||
first_put.proxy_headers.ssec_transport_present,
|
||||
"the replication PUT must have shipped the SSE-C transport headers the target then dropped"
|
||||
);
|
||||
assert!(
|
||||
requests.iter().any(|record| {
|
||||
record.operation == FakeTargetOperation::HeadObject
|
||||
&& record.key.as_deref() == Some("ssec-first.txt")
|
||||
&& record.sequence > first_put.sequence
|
||||
&& record.proxy_headers.replication_check.as_deref() == Some("true")
|
||||
}),
|
||||
"the post-PUT HEAD-back audit must have run through the replication-check channel; journal: {requests:?}"
|
||||
);
|
||||
|
||||
// Second SSE-C object: the cached Unsupported verdict fails it closed
|
||||
// before any PUT — including MRF retries of the first object.
|
||||
put_ssec("ssec-second.txt").await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "ssec-second.txt", "FAILED", true).await?;
|
||||
assert!(
|
||||
!target.requests().iter().any(|record| {
|
||||
record.operation == FakeTargetOperation::PutObject
|
||||
&& record.key.as_deref() != Some("plain-control.txt")
|
||||
&& record.proxy_headers.ssec_transport_present
|
||||
}),
|
||||
"no further SSE-C ciphertext may reach a target recorded Unsupported; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// The gate is scoped to SSE-C: plaintext replication keeps working.
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("plain-control.txt")
|
||||
.body(ByteStream::from_static(b"plaintext control payload"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "plain-control.txt", "COMPLETED", false).await?;
|
||||
assert!(target.has_object(target_bucket, "plain-control.txt"));
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// N2 (backlog#1675 P1-22): the admin replication-check must expose the same
|
||||
/// verdict operators would otherwise only learn from failing SSE-C objects —
|
||||
/// an SsecPassthrough probe phase that fails with the machine-readable
|
||||
/// `BucketRemoteSsecPassthroughUnsupported` code against a header-dropping
|
||||
/// target, with no probe residue left behind. The target's overall status
|
||||
/// stays OK: unlike version-identity drift, dropped passthrough headers are
|
||||
/// a capability limit, and a plaintext-only deployment against a MinIO-like
|
||||
/// target must not turn red.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "ssec-check-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "ssec-check-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
|
||||
assert_eq!(
|
||||
payload["Status"], "OK",
|
||||
"a capability-only SSE-C failure must not fail the check overall: {payload}"
|
||||
);
|
||||
let target_report = &payload["Targets"][0];
|
||||
assert_eq!(target_report["Status"], "OK", "{payload}");
|
||||
let ssec = &target_report["Phases"]["SsecPassthrough"];
|
||||
assert_eq!(ssec["Status"], "FAILED", "SsecPassthrough phase must fail: {payload}");
|
||||
assert_eq!(
|
||||
ssec["Code"], "BucketRemoteSsecPassthroughUnsupported",
|
||||
"the failure must carry the machine-readable code: {payload}"
|
||||
);
|
||||
// Basic replication of plaintext objects works on this target: every other
|
||||
// phase passes, so the code is the discriminator operators branch on.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
// The SSE-C probe PUT must have shipped the real transport header names —
|
||||
// a mangled or missing header set would fail the phase for the wrong
|
||||
// reason and mask a working target.
|
||||
let requests = target.requests();
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::PutObject && record.proxy_headers.ssec_transport_present),
|
||||
"the SSE-C probe PUT must carry the X-Rustfs-Replication-* transport headers; journal: {requests:?}"
|
||||
);
|
||||
|
||||
// No probe residue, including the SSE-C probe version.
|
||||
let probe_put = requests
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.ok_or("the probe PUT never reached the fake target")?;
|
||||
let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?;
|
||||
assert!(
|
||||
target.stored_versions(target_bucket, &probe_key).is_empty(),
|
||||
"all probe versions must be cleaned up"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// C1 (backlog#1675 P1-22): heal-path convergence for SSE-C. An SSE-C object
|
||||
/// whose live replication failed during a target outage must converge through
|
||||
/// the scanner/heal compensation once the target returns — passing the N2
|
||||
/// HEAD-back audit against the recovered RustFS target — and the replica must
|
||||
/// be readable with the customer key.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, mut target_env, source_bucket, target_bucket) =
|
||||
build_sse_replication_pair("ssec-heal", false, false).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let key = "ssec-heal-contract.txt";
|
||||
let body = b"repl-22 ssec heal payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
|
||||
// Target outage: the SSE-C write cannot replicate.
|
||||
target_env.stop_server();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// The failure is observable on the source (SSE-C HEAD needs the key).
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let head = source_client
|
||||
.head_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
match head.replication_status().map(|status| status.as_str()) {
|
||||
Some("PENDING") | Some("FAILED") => break,
|
||||
other => {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("source SSE-C object never reported PENDING/FAILED; last status={other:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recover the target in place; the source scanner re-drives the failure.
|
||||
target_env
|
||||
.restart_server_preserving_data(vec![], &[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")])
|
||||
.await?;
|
||||
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", true).await?;
|
||||
|
||||
// The healed replica is a REPLICA (status surfaces on HEAD) readable with
|
||||
// the customer key.
|
||||
let target_client = target_env.create_s3_client();
|
||||
let replica_head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica_head.replication_status().map(|status| status.as_str()),
|
||||
Some("REPLICA"),
|
||||
"the healed copy must carry REPLICA status"
|
||||
);
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// C1 (backlog#1675 P1-22): existing-object resync for SSE-C. An SSE-C object
|
||||
/// written BEFORE any replication config must reach the RustFS target through
|
||||
/// the existing-object resync (`replicate_all` transport, N2-audited), land as
|
||||
/// a REPLICA, and read back with the customer key.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||
source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
])
|
||||
.await?;
|
||||
|
||||
let source_bucket = "ssec-existing-src";
|
||||
let target_bucket = "ssec-existing-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
// The SSE-C object exists before any replication wiring.
|
||||
let key = "ssec-existing-contract.txt";
|
||||
let body = b"repl-22 ssec existing-object payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Wire replication (existing-object enabled) and drive a resync.
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
let (reset_arn, reset_id) = start_bucket_replication_reset(&source_env, source_bucket).await?;
|
||||
assert_eq!(reset_arn, target_arn);
|
||||
let terminal = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |status| {
|
||||
status.reset_id == reset_id && matches!(status.status.as_str(), "Completed" | "Failed")
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(terminal.status, "Completed", "SSE-C existing-object resync must complete");
|
||||
assert!(terminal.replicated_count >= 1, "the existing SSE-C object must have been resynced");
|
||||
|
||||
// The replica is a REPLICA (status surfaces on HEAD) readable with the
|
||||
// customer key.
|
||||
let replica_head = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica_head.replication_status().map(|status| status.as_str()),
|
||||
Some("REPLICA"),
|
||||
"the resynced copy must carry REPLICA status"
|
||||
);
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||
|
||||
// No plaintext leak: the replica stays unreadable without the key.
|
||||
assert!(
|
||||
target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.is_err(),
|
||||
"SSE-C replica must not be readable without the customer key"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#1147 repl-17 / backlog#1783: SSE-S3 objects replicate by decrypting
|
||||
/// at the source and re-encrypting on the target with the target's own KMS.
|
||||
/// The property backlog#1291 pinned — never a silent plaintext replica — still
|
||||
@@ -8417,3 +8824,304 @@ async fn test_scanner_never_compensates_when_existing_object_replication_disable
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shared setup for the P1-5 read-proxy scenarios (backlog#1675): a RustFS
|
||||
/// source with an enabled replication rule pointing at the fake target, and
|
||||
/// an object seeded DIRECTLY on the target — it exists remotely but not
|
||||
/// locally, exactly the active-active replication-lag window the read proxy
|
||||
/// serves.
|
||||
async fn start_read_proxy_lab(
|
||||
source_bucket: &str,
|
||||
target_bucket: &str,
|
||||
) -> Result<(FakeS3Target, RustFSTestEnvironment, Client, Client), Box<dyn Error + Send + Sync>> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(target_bucket);
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut process_env = replication_fast_env();
|
||||
process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
process_env.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
("RUST_LOG", "error"),
|
||||
]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &process_env).await?;
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let target_client = Client::from_conf(crate::common::build_test_s3_config(
|
||||
target.endpoint(),
|
||||
FAKE_ACCESS_KEY,
|
||||
FAKE_SECRET_KEY,
|
||||
None,
|
||||
"read-proxy-e2e",
|
||||
));
|
||||
|
||||
Ok((target, source_env, source_client, target_client))
|
||||
}
|
||||
|
||||
/// P1-5 (backlog#1675): during the active-active replication lag window a
|
||||
/// GET/HEAD for an object the local site does not have yet is proxied to the
|
||||
/// replication target. Pins the wire contract: the anti-loop
|
||||
/// `source-proxy-request` marker is sent, the replication worker's
|
||||
/// `source-replication-check` SSE-C exemption is NEVER sent, client SSE-C
|
||||
/// headers are forwarded verbatim, and an inbound request that was itself
|
||||
/// proxied is answered locally (404) without touching the target.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let source_bucket = "proxy-read-src";
|
||||
let target_bucket = "proxy-read-dst";
|
||||
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||
|
||||
let payload = b"proxy payload".to_vec();
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-only")
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
target.take_requests();
|
||||
|
||||
// a. GET of the locally-missing object is served through the proxy.
|
||||
let got = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied GET failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(got.content_length, Some(payload.len() as i64));
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), payload.as_slice(), "proxied GET must stream the target's body");
|
||||
|
||||
let get_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied GET")?;
|
||||
assert_eq!(
|
||||
get_record.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("true"),
|
||||
"proxied GET must carry the anti-loop source-proxy-request marker"
|
||||
);
|
||||
assert!(
|
||||
get_record.proxy_headers.replication_check.is_none(),
|
||||
"proxied GET must never carry the replication worker's source-replication-check exemption"
|
||||
);
|
||||
assert!(
|
||||
get_record.proxy_headers.ssec_algorithm.is_none() && !get_record.proxy_headers.ssec_key_present,
|
||||
"no client SSE-C headers were sent, so none may be forwarded"
|
||||
);
|
||||
|
||||
// a2. Client SSE-C headers travel verbatim to the target (the target owns
|
||||
// the real SSE-C decryption; the plaintext fake simply ignores them).
|
||||
target.take_requests();
|
||||
let ssec_key = "01234567890123456789012345678901";
|
||||
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
|
||||
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
|
||||
let _ = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&ssec_key_b64)
|
||||
.sse_customer_key_md5(&ssec_key_md5)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied SSE-C GET failed: {}", err.into_service_error()))?;
|
||||
let ssec_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied SSE-C GET")?;
|
||||
assert_eq!(ssec_record.proxy_headers.ssec_algorithm.as_deref(), Some("AES256"));
|
||||
assert!(ssec_record.proxy_headers.ssec_key_present, "SSE-C key header must be forwarded verbatim");
|
||||
assert_eq!(ssec_record.proxy_headers.ssec_key_md5.as_deref(), Some(ssec_key_md5.as_str()));
|
||||
assert!(ssec_record.proxy_headers.replication_check.is_none());
|
||||
|
||||
// b. HEAD of the locally-missing object is served through the proxy.
|
||||
target.take_requests();
|
||||
let head = source_client
|
||||
.head_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied HEAD failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(head.content_length, Some(payload.len() as i64));
|
||||
let head_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied HEAD")?;
|
||||
assert_eq!(head_record.proxy_headers.source_proxy_request.as_deref(), Some("true"));
|
||||
assert!(head_record.proxy_headers.replication_check.is_none());
|
||||
|
||||
// c. Anti-loop: an inbound request that already carries the proxy marker
|
||||
// is answered locally with 404 and never forwarded to the target.
|
||||
target.take_requests();
|
||||
let err = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-minio-source-proxy-request", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("anti-loop GET must fail locally instead of proxying");
|
||||
let service_err = err.into_service_error();
|
||||
assert!(service_err.is_no_such_key(), "anti-loop GET must 404, got: {service_err}");
|
||||
assert!(
|
||||
!target
|
||||
.requests()
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||
"anti-loop GET must not reach the replication target; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// c2. MinIO ProxyHeaderSet parity: the header's mere PRESENCE disables
|
||||
// proxying — "false" is exactly what a peer's replication worker sends on
|
||||
// its convergence HEADs, and proxying that miss back would fake
|
||||
// convergence.
|
||||
target.take_requests();
|
||||
let err = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-minio-source-proxy-request", "false");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("proxy-header-set GET must fail locally instead of proxying");
|
||||
let service_err = err.into_service_error();
|
||||
assert!(service_err.is_no_such_key(), "proxy-header-set GET must 404, got: {service_err}");
|
||||
assert!(
|
||||
!target
|
||||
.requests()
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||
"proxy-header-set GET must not reach the replication target; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// d. The replication worker's own convergence HEAD against the target
|
||||
// must carry `source-proxy-request: false` (never proxied back) and the
|
||||
// replication-check exemption. Trigger real replication and inspect the
|
||||
// fake journal.
|
||||
target.take_requests();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("worker-replicated")
|
||||
.body(ByteStream::from_static(b"worker payload"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "worker-replicated").await?;
|
||||
let worker_head = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("worker-replicated"))
|
||||
.ok_or_else(|| format!("replication worker never HEAD-ed the target; journal: {:?}", target.requests()))?;
|
||||
assert_eq!(
|
||||
worker_head.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("false"),
|
||||
"worker convergence HEAD must send source-proxy-request: false so the target answers locally"
|
||||
);
|
||||
assert_eq!(
|
||||
worker_head.proxy_headers.replication_check.as_deref(),
|
||||
Some("true"),
|
||||
"worker convergence HEAD keeps the replication-check exemption"
|
||||
);
|
||||
|
||||
drop(source_env);
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-5 (backlog#1675): GetObjectTagging for an object missing locally is
|
||||
/// proxied to the replication target with the anti-loop marker, mirroring
|
||||
/// MinIO `proxyGetTaggingToRepTarget`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let source_bucket = "proxy-tag-src";
|
||||
let target_bucket = "proxy-tag-dst";
|
||||
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-tagged")
|
||||
.body(ByteStream::from_static(b"tagged payload"))
|
||||
.send()
|
||||
.await?;
|
||||
target_client
|
||||
.put_object_tagging()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-tagged")
|
||||
.tagging(
|
||||
aws_sdk_s3::types::Tagging::builder()
|
||||
.tag_set(aws_sdk_s3::types::Tag::builder().key("team").value("storage").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
target.take_requests();
|
||||
|
||||
let tags = source_client
|
||||
.get_object_tagging()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-tagged")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied GetObjectTagging failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(tags.tag_set.len(), 1, "proxied tagging read must return the target's tags");
|
||||
assert_eq!(tags.tag_set[0].key.as_str(), "team");
|
||||
assert_eq!(tags.tag_set[0].value.as_str(), "storage");
|
||||
|
||||
let record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObjectTagging && record.key.as_deref() == Some("proxy-tagged"))
|
||||
.ok_or("fake target never received the proxied GetObjectTagging")?;
|
||||
assert_eq!(
|
||||
record.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("true"),
|
||||
"proxied tagging read must carry the anti-loop marker"
|
||||
);
|
||||
assert!(record.proxy_headers.replication_check.is_none());
|
||||
|
||||
drop(source_env);
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
TargetClient, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -131,16 +131,19 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -182,24 +185,26 @@ pub mod bucket {
|
||||
mrf_backlog_observability_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -276,7 +281,9 @@ pub mod cluster {
|
||||
}
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled};
|
||||
pub use crate::io_support::compress::{
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod config {
|
||||
@@ -316,7 +323,7 @@ pub mod data_usage {
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||
@@ -403,8 +410,11 @@ pub mod metrics {
|
||||
}
|
||||
|
||||
pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
@@ -464,7 +474,8 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,10 +27,15 @@ use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
use aws_sdk_s3::operation::delete_object_tagging::{DeleteObjectTaggingError, DeleteObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
};
|
||||
@@ -57,8 +62,10 @@ use rustfs_utils::http::{
|
||||
is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -80,7 +87,6 @@ use tracing::warn;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
@@ -293,9 +299,51 @@ struct TargetClientBuildProbe {
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
/// Whether a replication target preserves the SSE-C passthrough transport
|
||||
/// headers (`X-Rustfs-Replication-*`) end to end.
|
||||
///
|
||||
/// A target that silently drops those headers (MinIO, generic S3) stores the
|
||||
/// forwarded ciphertext without its decryption material — an unreadable
|
||||
/// replica that used to report COMPLETED. The replication worker audits the
|
||||
/// first passthrough PUT per target (HEAD-back for SSE-C evidence) and caches
|
||||
/// the verdict here; a fresh `Unsupported` fails SSE-C replication closed
|
||||
/// before any PUT is sent. Entries follow the `arn_remotes_map` lifecycle
|
||||
/// (rebuilding or removing a target resets its capability to `Unknown`) and
|
||||
/// additionally expire after [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], after which
|
||||
/// the next attempt re-audits.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum SsecPassthroughCapability {
|
||||
#[default]
|
||||
Unknown,
|
||||
Supported,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
/// Trade-off: without a TTL a verdict is sticky for the process lifetime —
|
||||
/// an `Unsupported` target that gets upgraded (or re-probed only via
|
||||
/// replication-check) would keep failing SSE-C replication forever, and the
|
||||
/// fail-open twin: a `Supported` verdict would outlive a backend swapped
|
||||
/// behind the same endpoint/ARN. With the TTL, a bad target costs at most
|
||||
/// one wasted PUT+HEAD audit per TTL window, and a changed backend is
|
||||
/// re-discovered within the same window.
|
||||
pub const SSEC_PASSTHROUGH_CAPABILITY_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
/// A recorded SSE-C passthrough verdict plus when it was recorded, so reads
|
||||
/// can report staleness against [`SSEC_PASSTHROUGH_CAPABILITY_TTL`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct SsecPassthroughRecord {
|
||||
capability: SsecPassthroughCapability,
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
@@ -316,6 +364,7 @@ impl BucketTargetSys {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -579,19 +628,59 @@ impl BucketTargetSys {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
|
||||
if let Some(targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
/// Staleness is computed here so the gate policy stays a pure function.
|
||||
pub async fn ssec_passthrough_capability(&self, arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
match self.ssec_passthrough_map.read().await.get(arn) {
|
||||
Some(record) => (record.capability, record.recorded_at.elapsed() >= SSEC_PASSTHROUGH_CAPABILITY_TTL),
|
||||
None => (SsecPassthroughCapability::Unknown, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record an audited SSE-C passthrough verdict for a target ARN. Written by
|
||||
/// the replication worker's HEAD-back audit and by the replication-check
|
||||
/// SsecPassthrough probe phase.
|
||||
pub async fn record_ssec_passthrough_capability(&self, arn: &str, capability: SsecPassthroughCapability) {
|
||||
self.ssec_passthrough_map.write().await.insert(
|
||||
arn.to_string(),
|
||||
SsecPassthroughRecord {
|
||||
capability,
|
||||
recorded_at: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Test hook: age an existing verdict so TTL expiry is observable without
|
||||
/// waiting out the real window.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn backdate_ssec_passthrough_capability(&self, arn: &str, age: Duration) {
|
||||
let backdated = Instant::now()
|
||||
.checked_sub(age)
|
||||
.expect("system uptime must exceed the backdate age");
|
||||
if let Some(record) = self.ssec_passthrough_map.write().await.get_mut(arn) {
|
||||
record.recorded_at = backdated;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set_target(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -947,15 +1036,21 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1445,6 +1540,43 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the S3 `versionId` for a proxied read against a remote target.
|
||||
/// RustFS represents the null version internally as the nil UUID while the S3
|
||||
/// API addresses it as the literal "null" (same mapping as
|
||||
/// [`resolve_put_api_version_id`]); empty means "no version requested".
|
||||
pub(crate) fn resolve_read_api_version_id(version_id: Option<String>) -> Option<String> {
|
||||
let version_id = version_id?;
|
||||
let trimmed = version_id.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else if Uuid::parse_str(trimmed).is_ok_and(|uuid| uuid.is_nil()) {
|
||||
Some(rustfs_filemeta::NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Outbound header set for a proxied read: the caller-provided passthrough
|
||||
/// headers (client SSE-C key family, conditional headers) plus the anti-loop
|
||||
/// `source-proxy-request` marker in both the x-rustfs- and x-minio- prefixes
|
||||
/// (a MinIO target only understands the latter). Never adds
|
||||
/// `source-replication-check`: that exemption channel belongs exclusively to
|
||||
/// the replication worker's HEAD.
|
||||
fn proxy_outbound_headers(mut extra_headers: HeaderMap) -> HeaderMap {
|
||||
insert_header(&mut extra_headers, SUFFIX_SOURCE_PROXY_REQUEST, "true");
|
||||
extra_headers
|
||||
}
|
||||
|
||||
/// Copy `headers` onto an SDK request inside `customize().map_request` (runs
|
||||
/// before signing, so the headers join the SigV4 canonical request).
|
||||
fn apply_extra_headers(mut req: HttpRequest, headers: &HeaderMap) -> Result<HttpRequest, std::convert::Infallible> {
|
||||
for (k, v) in headers.iter() {
|
||||
req.headers_mut()
|
||||
.insert(k.as_str().to_string(), v.to_str().unwrap_or("").to_string());
|
||||
}
|
||||
Ok(req)
|
||||
}
|
||||
|
||||
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
||||
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
@@ -1476,9 +1608,12 @@ impl Default for AdvancedPutOptions {
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
source_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
retention_timestamp: OffsetDateTime::now_utc(),
|
||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||
// timestamp header for it, otherwise a receiver would treat an
|
||||
// unset category as a modification made right now.
|
||||
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
@@ -1675,6 +1810,16 @@ impl PutObjectOptions {
|
||||
);
|
||||
}
|
||||
|
||||
for (suffix, timestamp) in [
|
||||
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||
] {
|
||||
if timestamp.unix_timestamp() != 0 {
|
||||
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
|
||||
if self.internal.replication_request {
|
||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
@@ -1837,6 +1982,13 @@ impl TargetClient {
|
||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
// `source-proxy-request: false` (MinIO `ProxyHeaderSet` semantics):
|
||||
// the header's mere presence tells the receiver to answer LOCALLY
|
||||
// instead of proxying the miss back to us. Without it, a not-found on
|
||||
// the target gets read-proxied back to this source, echoes the source
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
match self
|
||||
.client
|
||||
.head_object()
|
||||
@@ -1861,6 +2013,129 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
/// Deliberately different from [`TargetClient::head_object`]: it must NOT
|
||||
/// send `source-replication-check` — that header is the replication
|
||||
/// worker's SSE-C metadata exemption channel. A proxied client request
|
||||
/// instead forwards the client's own SSE-C headers (`extra_headers`) so
|
||||
/// the target performs the real SSE-C validation/decryption. The
|
||||
/// `source-proxy-request` marker is always added so the target does not
|
||||
/// proxy the request onward (anti-loop).
|
||||
pub async fn head_object_for_proxy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||
/// Returns the streaming SDK output; callers must forward the body without
|
||||
/// buffering it. Same header contract as [`Self::head_object_for_proxy`]:
|
||||
/// anti-loop marker on, replication-check never sent, client SSE-C /
|
||||
/// conditional headers forwarded verbatim via `extra_headers`.
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GetObjectTagging for the tagging read-proxy path
|
||||
/// (MinIO `proxyGetTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn get_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// PutObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn put_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
tagging: SdkTagging,
|
||||
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.tagging(tagging)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn delete_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.delete_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
@@ -2490,6 +2765,57 @@ mod tests {
|
||||
assert_eq!(health.last_online, Some(now));
|
||||
}
|
||||
|
||||
/// N2 TTL contract, both flip directions: a recorded verdict is fresh
|
||||
/// until [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], then reads as expired; a
|
||||
/// re-audit that records the OPPOSITE verdict replaces it as fresh. The
|
||||
/// worker gate maps expired verdicts to ProceedWithAudit (pinned in
|
||||
/// `replication_target_boundary`), so together this proves an Unsupported
|
||||
/// target recovers to Supported through the audit once its verdict ages
|
||||
/// out — and a stale Supported one is re-proven rather than trusted.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_capability_ttl_expires_and_reaudit_flips_verdict() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:ssec-ttl";
|
||||
let expired_age = SSEC_PASSTHROUGH_CAPABILITY_TTL + Duration::from_secs(1);
|
||||
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unknown, false),
|
||||
"an unrecorded target must read Unknown and never expired"
|
||||
);
|
||||
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Unsupported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, false)
|
||||
);
|
||||
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, true),
|
||||
"an aged-out Unsupported verdict must read expired so the gate re-audits"
|
||||
);
|
||||
|
||||
// The re-audit against an upgraded target records Supported afresh.
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Supported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, false),
|
||||
"a fresh Supported verdict replaces the expired Unsupported one"
|
||||
);
|
||||
|
||||
// And the fail-open twin: Supported also ages out.
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, true),
|
||||
"an aged-out Supported verdict must read expired so the gate re-proves it"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
||||
let sys = BucketTargetSys::default();
|
||||
@@ -2842,6 +3168,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||
// last-writer-wins on these headers (object-api-options.go parses them
|
||||
// as RFC3339); a replica without them loses every conflict resolution.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||
opts.internal.tagging_timestamp = tagging;
|
||||
opts.internal.retention_timestamp = retention;
|
||||
opts.internal.legalhold_timestamp = legalhold;
|
||||
|
||||
let header = opts.header();
|
||||
for (suffix, expected) in [
|
||||
("source-replication-tagging-timestamp", tagging),
|
||||
("source-replication-retention-timestamp", retention),
|
||||
("source-replication-legalhold-timestamp", legalhold),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||
"replication put requests must carry the {suffix} header"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||
// make the receiver treat an unset category as a fresh modification.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
|
||||
let header = opts.header();
|
||||
for suffix in [
|
||||
"source-replication-tagging-timestamp",
|
||||
"source-replication-retention-timestamp",
|
||||
"source-replication-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||
"unset {suffix} must not be sent to replication targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
@@ -46,15 +46,13 @@ use crate::bucket::lifecycle::transition_transaction::run_transition_transaction
|
||||
use crate::bucket::object_lock::ObjectLockApi;
|
||||
use crate::bucket::versioning::VersioningApi as _;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::object_api_utils::new_getobjectreader;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
|
||||
use crate::error::Error;
|
||||
use crate::error::StorageError;
|
||||
use crate::error::{
|
||||
error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down,
|
||||
};
|
||||
use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
|
||||
use crate::services::tier::{
|
||||
tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata},
|
||||
warm_backend::WarmBackendGetOpts,
|
||||
@@ -128,11 +126,23 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
|
||||
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
||||
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
||||
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
||||
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
||||
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub type TraceFn =
|
||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
@@ -142,9 +152,21 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
||||
|
||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||
@@ -282,6 +304,10 @@ impl LifecycleSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||
let bucket = oi.bucket.clone();
|
||||
let name = oi.name.clone();
|
||||
@@ -572,6 +598,10 @@ async fn delete_free_version_remote_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
@@ -2870,12 +2900,37 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
||||
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||
}
|
||||
|
||||
async fn stale_upload_current_size_with_opts(
|
||||
set: &Arc<SetDisks>,
|
||||
metadata: &HashMap<String, String>,
|
||||
upload_dir: &str,
|
||||
no_lock: bool,
|
||||
) -> Option<usize> {
|
||||
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
||||
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
||||
let upload_id = encode_stale_upload_id(upload_dir);
|
||||
let data_movement = rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||
let parts = set
|
||||
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.list_object_parts(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
None,
|
||||
MAX_PARTS_COUNT,
|
||||
&ObjectOptions {
|
||||
data_movement,
|
||||
no_lock,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()?;
|
||||
|
||||
@@ -2893,7 +2948,12 @@ async fn stale_upload_lifecycle_due(
|
||||
metadata: &HashMap<String, String>,
|
||||
initiated: OffsetDateTime,
|
||||
upload_dir: &str,
|
||||
no_lock: bool,
|
||||
) -> Option<OffsetDateTime> {
|
||||
if rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
||||
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
||||
|
||||
@@ -2906,7 +2966,9 @@ async fn stale_upload_lifecycle_due(
|
||||
name: object.clone(),
|
||||
user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
|
||||
mod_time: Some(initiated),
|
||||
size: stale_upload_current_size(set, metadata, upload_dir).await.unwrap_or_default(),
|
||||
size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
|
||||
.await
|
||||
.unwrap_or_default(),
|
||||
is_latest: true,
|
||||
delete_marker: false,
|
||||
user_defined: metadata.clone(),
|
||||
@@ -2934,6 +2996,7 @@ async fn read_stale_multipart_candidate(
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: false,
|
||||
},
|
||||
) {
|
||||
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
|
||||
@@ -2973,36 +3036,30 @@ fn merge_stale_multipart_candidate(
|
||||
}
|
||||
}
|
||||
|
||||
fn is_multipart_sha_dir(path: &str) -> bool {
|
||||
path.len() == 64 && path.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
fn multipart_sha_path(root: &str, entry: &str) -> Option<String> {
|
||||
let sha_dir = entry.trim_end_matches('/');
|
||||
is_multipart_sha_dir(sha_dir).then(|| {
|
||||
if root.is_empty() {
|
||||
sha_dir.to_string()
|
||||
} else {
|
||||
format!("{root}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
||||
for disk in set.get_local_disks().await.into_iter().flatten() {
|
||||
if !disk.is_online().await {
|
||||
continue;
|
||||
}
|
||||
|
||||
let sha_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
error = ?err,
|
||||
reason = "multipart_root_list_failed",
|
||||
"Skipped empty multipart sha cleanup"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for sha_dir in sha_dirs {
|
||||
let sha_dir = sha_dir.trim_end_matches('/').to_string();
|
||||
let upload_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let sha_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
@@ -3012,9 +3069,8 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_sha_dir_list_failed",
|
||||
reason = "multipart_root_list_failed",
|
||||
"Skipped empty multipart sha cleanup"
|
||||
);
|
||||
}
|
||||
@@ -3022,25 +3078,48 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
||||
}
|
||||
};
|
||||
|
||||
if !upload_dirs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
|
||||
let upload_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_sha_dir_list_failed",
|
||||
"Skipped empty multipart sha cleanup"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = disk
|
||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
|
||||
.await
|
||||
&& err != DiskError::FileNotFound
|
||||
&& err != DiskError::VolumeNotFound
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_sha_dir_remove_failed",
|
||||
"Failed to remove empty multipart sha dir"
|
||||
);
|
||||
if !upload_dirs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Err(err) = disk
|
||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
|
||||
.await
|
||||
&& err != DiskError::FileNotFound
|
||||
&& err != DiskError::VolumeNotFound
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_sha_dir_remove_failed",
|
||||
"Failed to remove empty multipart sha dir"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3058,30 +3137,9 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
||||
continue;
|
||||
}
|
||||
|
||||
let sha_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
error = ?err,
|
||||
reason = "multipart_root_list_failed",
|
||||
"Skipped stale multipart cleanup"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for sha_dir in sha_dirs {
|
||||
let sha_dir = sha_dir.trim_end_matches('/').to_string();
|
||||
let upload_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let sha_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => entries,
|
||||
@@ -3091,9 +3149,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_sha_dir_list_failed",
|
||||
reason = "multipart_root_list_failed",
|
||||
"Skipped stale multipart cleanup"
|
||||
);
|
||||
}
|
||||
@@ -3101,39 +3158,62 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
||||
}
|
||||
};
|
||||
|
||||
for upload_dir in upload_dirs {
|
||||
let upload_dir = upload_dir.trim_end_matches('/').to_string();
|
||||
let candidate_path = format!("{sha_dir}/{upload_dir}");
|
||||
if candidates
|
||||
.get(&candidate_path)
|
||||
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
|
||||
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
|
||||
let upload_dirs = match disk
|
||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||
.await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
|
||||
Ok(candidate) => candidate,
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err != DiskError::FileNotFound {
|
||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
path = %candidate_path,
|
||||
sha_dir = %sha_dir,
|
||||
error = ?err,
|
||||
reason = "multipart_metadata_read_failed",
|
||||
"Multipart metadata unavailable during stale cleanup"
|
||||
reason = "multipart_sha_dir_list_failed",
|
||||
"Skipped stale multipart cleanup"
|
||||
);
|
||||
}
|
||||
let initiated = initiated_from_upload_dir(&upload_dir, None);
|
||||
StaleMultipartUploadCandidate {
|
||||
path: candidate_path,
|
||||
initiated,
|
||||
metadata: None,
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
merge_stale_multipart_candidate(&mut candidates, candidate);
|
||||
|
||||
for upload_dir in upload_dirs {
|
||||
let upload_dir = upload_dir.trim_end_matches('/').to_string();
|
||||
let candidate_path = format!("{sha_dir}/{upload_dir}");
|
||||
if candidates
|
||||
.get(&candidate_path)
|
||||
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
|
||||
Ok(candidate) => candidate,
|
||||
Err(err) => {
|
||||
if err != DiskError::FileNotFound {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
path = %candidate_path,
|
||||
error = ?err,
|
||||
reason = "multipart_metadata_read_failed",
|
||||
"Multipart metadata unavailable during stale cleanup"
|
||||
);
|
||||
}
|
||||
let initiated = initiated_from_upload_dir(&upload_dir, None);
|
||||
StaleMultipartUploadCandidate {
|
||||
path: candidate_path,
|
||||
initiated,
|
||||
metadata: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
merge_stale_multipart_candidate(&mut candidates, candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3142,7 +3222,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
||||
let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
|
||||
let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
|
||||
if let Some(metadata) = candidate.metadata.as_ref()
|
||||
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir).await
|
||||
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
|
||||
&& lifecycle_due < due
|
||||
{
|
||||
due = lifecycle_due;
|
||||
@@ -3152,34 +3232,49 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
||||
continue;
|
||||
}
|
||||
|
||||
match set.delete_all(RUSTFS_META_MULTIPART_BUCKET, &candidate.path).await {
|
||||
let cleanup_guard = match set.lock_stale_multipart_cleanup(&candidate.path).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
path = %candidate.path,
|
||||
error = ?err,
|
||||
reason = "multipart_cleanup_lock_or_recheck_failed",
|
||||
"Skipped stale multipart cleanup"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let current_metadata = cleanup_guard.file_info().metadata.clone();
|
||||
let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
|
||||
let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
|
||||
if let Some(lifecycle_due) =
|
||||
stale_upload_lifecycle_due(set, ¤t_metadata, current_initiated, &upload_dir, true).await
|
||||
&& lifecycle_due < current_due
|
||||
{
|
||||
current_due = lifecycle_due;
|
||||
}
|
||||
if now < current_due || cleanup_guard.is_lock_lost() {
|
||||
continue;
|
||||
}
|
||||
|
||||
match cleanup_guard.delete(set).await {
|
||||
Ok(()) => {
|
||||
deleted += 1;
|
||||
let upload_id = encode_stale_upload_id(&upload_dir);
|
||||
if let Some(metadata) = candidate.metadata.as_ref() {
|
||||
debug!(
|
||||
bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
|
||||
object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
|
||||
upload_id = %upload_id,
|
||||
due = ?due,
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "removed",
|
||||
"Removed stale multipart upload"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
path = %candidate.path,
|
||||
upload_id = %upload_id,
|
||||
due = ?due,
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "removed",
|
||||
"Removed stale multipart upload"
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
bucket = current_metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
|
||||
object = current_metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
|
||||
upload_id = %upload_id,
|
||||
due = ?current_due,
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "removed",
|
||||
"Removed stale multipart upload"
|
||||
);
|
||||
}
|
||||
Err(err) => debug!(
|
||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||
@@ -3293,6 +3388,10 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
||||
if remote_delete_succeeded {
|
||||
opts.skip_decommissioned = true;
|
||||
@@ -4280,6 +4379,10 @@ pub async fn expire_transitioned_object(
|
||||
Ok(dobj)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
let us = Uuid::new_v4().to_string();
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -4314,6 +4417,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
let tier = _tier.to_string();
|
||||
Arc::new(move || {
|
||||
@@ -4332,6 +4439,10 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_transitioned_object_reader(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -4339,9 +4450,10 @@ pub async fn get_transitioned_object_reader(
|
||||
h: &HeaderMap,
|
||||
oi: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
resolver: Option<&dyn ObjectEncryptionResolver>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
let tier_config_mgr = runtime_sources::tier_config_mgr_handle();
|
||||
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await
|
||||
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr, resolver).await
|
||||
}
|
||||
|
||||
fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
@@ -4361,6 +4473,10 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver joins the tier manager as the second injected port this read
|
||||
// needs; grouping the request half into a struct would churn every call site of
|
||||
// a bug fix.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -4369,6 +4485,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
oi: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
resolver: Option<&dyn ObjectEncryptionResolver>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
@@ -4386,11 +4503,16 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
|
||||
let ret = new_getobjectreader(rs, oi, opts, h);
|
||||
if let Err(err) = ret {
|
||||
return Err(error_resp_to_object_err(err, vec![bucket, object]));
|
||||
}
|
||||
let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check");
|
||||
// The same read plan the local path uses, so the tier fetch is positioned in
|
||||
// the object's *stored* coordinate system and the stream is handed the same
|
||||
// decrypt/decompress transforms. Reading an encrypted object's ciphertext
|
||||
// through a plaintext-coordinate range and skipping the transform is how a
|
||||
// transitioned SSE object used to come back as silently corrupt bytes of the
|
||||
// right length (rustfs/rustfs#6025).
|
||||
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
|
||||
let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
|
||||
let mut gopts = WarmBackendGetOpts::default();
|
||||
|
||||
if off >= 0 && length >= 0 {
|
||||
@@ -4427,7 +4549,10 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
);
|
||||
e
|
||||
})?;
|
||||
Ok(attach_tier_operation_lease(get_fn(reader, h.clone()), tgt_client))
|
||||
let object_reader = plan
|
||||
.into_object_reader(Box::new(reader), oi)
|
||||
.map_err(|err| std::io::Error::other(format!("wrapping the tier stream for {bucket}/{object} failed: {err}")))?;
|
||||
Ok(attach_tier_operation_lease(object_reader, tgt_client))
|
||||
}
|
||||
|
||||
struct TierOperationLeaseReader {
|
||||
@@ -5072,6 +5197,10 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
|
||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||
let mut success = false;
|
||||
match event.action {
|
||||
@@ -5162,6 +5291,7 @@ mod tests {
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::{
|
||||
@@ -5714,6 +5844,7 @@ mod tests {
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("transitioned reader should open");
|
||||
@@ -5778,6 +5909,7 @@ mod tests {
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -5818,6 +5950,7 @@ mod tests {
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -6055,6 +6188,7 @@ mod tests {
|
||||
&oi,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -6078,6 +6212,7 @@ mod tests {
|
||||
&oi,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -7343,6 +7478,10 @@ mod tests {
|
||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
||||
#[allow(unsafe_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
|
||||
)]
|
||||
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
@@ -11926,6 +12065,135 @@ mod tests {
|
||||
assert!(is_err_invalid_upload_id(&err));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_multipart_cleanup_handles_data_movement_namespace() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-data-movement-{}", Uuid::new_v4().simple());
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let create_upload = |object: &'static str, mod_time| {
|
||||
let ecstore = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
async move {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
"cleanup-test".to_string(),
|
||||
);
|
||||
ecstore
|
||||
.new_multipart_upload(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
mod_time: Some(mod_time),
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("data movement multipart upload should be created")
|
||||
.upload_id
|
||||
}
|
||||
};
|
||||
|
||||
let stale_object = "stale-internal.bin";
|
||||
let active_object = "active-internal.bin";
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let stale_upload_id = create_upload(stale_object, now - time::Duration::hours(30)).await;
|
||||
let active_upload_id = create_upload(active_object, now).await;
|
||||
|
||||
let deleted = cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60)).await;
|
||||
assert!(deleted >= 1, "expected stale data movement upload to be removed");
|
||||
|
||||
let internal_opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
};
|
||||
let stale_err = ecstore
|
||||
.get_multipart_info(&bucket, stale_object, &stale_upload_id, &internal_opts)
|
||||
.await
|
||||
.expect_err("stale data movement upload should be removed");
|
||||
assert!(is_err_invalid_upload_id(&stale_err));
|
||||
ecstore
|
||||
.get_multipart_info(&bucket, active_object, &active_upload_id, &internal_opts)
|
||||
.await
|
||||
.expect("active data movement upload should remain available");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_multipart_cleanup_waits_for_data_movement_part_commit() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-data-movement-lock-{}", Uuid::new_v4().simple());
|
||||
let object = "stale-internal.bin";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
"cleanup-lock-test".to_string(),
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::hours(30)),
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
};
|
||||
let upload = ecstore
|
||||
.new_multipart_upload(&bucket, object, &opts)
|
||||
.await
|
||||
.expect("data movement multipart upload should be created");
|
||||
let barrier = MultipartCommitBarrier::install(bucket.as_str(), object, MultipartCommitPause::PutPartAfterRename);
|
||||
let put_store = ecstore.clone();
|
||||
let put_bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let put_task = tokio::spawn(async move {
|
||||
let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
|
||||
put_store
|
||||
.put_object_part(
|
||||
&put_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
1,
|
||||
&mut data,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let cleanup_store = ecstore.clone();
|
||||
let mut cleanup_task = tokio::spawn(async move {
|
||||
cleanup_stale_multipart_uploads_once_at(
|
||||
cleanup_store,
|
||||
OffsetDateTime::now_utc(),
|
||||
StdDuration::from_secs(24 * 60 * 60),
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(200), &mut cleanup_task)
|
||||
.await
|
||||
.is_err(),
|
||||
"stale cleanup must wait for the in-flight part commit upload lock"
|
||||
);
|
||||
|
||||
barrier.release();
|
||||
put_task
|
||||
.await
|
||||
.expect("part upload task should join")
|
||||
.expect("part upload should commit before stale cleanup");
|
||||
let deleted = cleanup_task.await.expect("stale cleanup task should join");
|
||||
assert!(deleted >= 1, "stale cleanup should proceed after the part commit releases its lock");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_multipart_cleanup_applies_abort_incomplete_lifecycle_before_default_expiry() {
|
||||
@@ -12000,6 +12268,58 @@ mod tests {
|
||||
assert!(is_err_invalid_upload_id(&err));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_multipart_cleanup_excludes_data_movement_from_abort_lifecycle() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-internal-lifecycle-{}", Uuid::new_v4().simple());
|
||||
let object = "logs/internal/object.bin";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
|
||||
|
||||
let initiated = OffsetDateTime::now_utc() - time::Duration::minutes(5);
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
"lifecycle-exclusion-test".to_string(),
|
||||
);
|
||||
let upload = ecstore
|
||||
.new_multipart_upload(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
mod_time: Some(initiated),
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("data movement multipart upload should be created");
|
||||
|
||||
let deleted = cleanup_stale_multipart_uploads_once_at(
|
||||
ecstore.clone(),
|
||||
OffsetDateTime::now_utc(),
|
||||
StdDuration::from_secs(7 * 24 * 60 * 60),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(deleted, 0, "bucket lifecycle must not remove active data movement uploads");
|
||||
|
||||
ecstore
|
||||
.get_multipart_info(
|
||||
&bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active data movement upload should remain available");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn stale_multipart_cleanup_applies_abort_lifecycle_with_size_filter() {
|
||||
|
||||
@@ -759,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord {
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultRecord {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||
Self::new_with_reason(job_id, task_key, result, None)
|
||||
}
|
||||
@@ -1257,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_task_record(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1320,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_worker_result_stats(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1455,6 +1467,10 @@ async fn scan_manual_transition_worker_result_journal(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn reconcile_manual_transition_worker_results(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
|
||||
@@ -15,25 +15,35 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||
obj.version_purge_status.is_pending()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,10 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||
}
|
||||
|
||||
@@ -331,6 +331,10 @@ where
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -148,6 +148,10 @@ struct RecoveryCursor {
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn recover_tier_free_versions(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
|
||||
@@ -385,6 +385,10 @@ impl ExpiryOp for Jentry {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||
if let Err(err) = &result
|
||||
@@ -395,6 +399,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
#[cfg(test)]
|
||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||
@@ -405,6 +413,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -485,6 +497,10 @@ pub enum RemoteTierDeleteOutcome {
|
||||
AlreadyRemoved,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
|
||||
@@ -50,8 +50,16 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionTransactionError {
|
||||
#[error("transition transaction already exists")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
AlreadyExists,
|
||||
#[error("transition transaction is not found")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
NotFound,
|
||||
#[error("transition transaction is corrupt: {0}")]
|
||||
Corrupt(&'static str),
|
||||
|
||||
@@ -50,6 +50,75 @@ use uuid::Uuid;
|
||||
|
||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
struct ConfigWriteLockProbeState {
|
||||
bucket: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub struct ConfigWriteLockProbe {
|
||||
state: Arc<ConfigWriteLockProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl ConfigWriteLockProbe {
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(ConfigWriteLockProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
assert!(
|
||||
!probes.iter().any(|current| current.bucket == state.bucket),
|
||||
"config write lock probe must be unique for a bucket"
|
||||
);
|
||||
probes.push(Arc::clone(&state));
|
||||
drop(probes);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub async fn wait_until_attempted(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("bucket config update should attempt the transaction lock");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for ConfigWriteLockProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
probes.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_config_write_lock_attempt(bucket: &str) {
|
||||
let probe = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison")
|
||||
.iter()
|
||||
.find(|probe| probe.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(probe) = probe {
|
||||
probe.arrived.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MetadataLoadMode {
|
||||
Initial,
|
||||
@@ -590,6 +659,41 @@ pub async fn update_under_transaction_lock(
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||
pub async fn delete_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||
}
|
||||
|
||||
pub async fn update_quota_if_incarnation(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||
sys.clone(),
|
||||
bucket,
|
||||
Some(expected_incarnation_id),
|
||||
))
|
||||
.await?;
|
||||
if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
|
||||
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
bucket: bucket.to_string(),
|
||||
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
@@ -704,6 +808,14 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||
}
|
||||
|
||||
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
||||
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -734,7 +846,26 @@ async fn acquire_transaction_lock_with_sys(
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
tokio::pin!(acquire);
|
||||
let mut notified = false;
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if !notified {
|
||||
notify_config_write_lock_attempt(bucket);
|
||||
notified = true;
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await?;
|
||||
Ok(guard)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
Ok(acquire.await?)
|
||||
}
|
||||
|
||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||
@@ -762,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -889,6 +1024,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read the quota configuration and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the bucket metadata
|
||||
/// transaction read lock.
|
||||
pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
metadata.quota_config_updated_at,
|
||||
))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -945,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1458,6 +1628,7 @@ impl BucketMetadataSys {
|
||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||
/// instead of supplied up front. Loads through this system's own store so
|
||||
/// the read and the persisted write target the same instance.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
@@ -1562,6 +1733,7 @@ impl BucketMetadataSys {
|
||||
/// A miss is never published as an authoritative default, and a snapshot
|
||||
/// read before delete plus same-name recreation cannot replace the new
|
||||
/// generation.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
|
||||
@@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
||||
|
||||
/// Check if an object has legal hold enabled.
|
||||
/// Returns true if legal hold is ON.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||
@@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
||||
/// # Returns
|
||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||
/// * `false` if the object is not locked
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||
// Delete markers are never locked
|
||||
if is_delete_marker {
|
||||
|
||||
@@ -52,6 +52,7 @@ impl QuotaChecker {
|
||||
) -> Result<QuotaCheckResult, QuotaError> {
|
||||
let start_time = Instant::now();
|
||||
let quota_config = self.get_quota_config(bucket).await?;
|
||||
let uses_durable_reservations = quota_config.uses_durable_reservations();
|
||||
|
||||
// If no quota limit is set, allow operation
|
||||
let quota_limit = match quota_config.quota {
|
||||
@@ -67,6 +68,7 @@ impl QuotaChecker {
|
||||
quota_limit: None,
|
||||
operation_size,
|
||||
remaining: None,
|
||||
uses_durable_reservations,
|
||||
});
|
||||
}
|
||||
Some(q) => q,
|
||||
@@ -74,14 +76,17 @@ impl QuotaChecker {
|
||||
|
||||
let current_usage = self.get_real_time_usage(bucket).await?;
|
||||
|
||||
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
|
||||
let expected_usage = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
current_usage.saturating_add(admission_size)
|
||||
}
|
||||
QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
|
||||
};
|
||||
|
||||
let allowed = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
quota_config.check_operation_allowed(current_usage, operation_size)
|
||||
quota_config.check_operation_allowed(current_usage, admission_size)
|
||||
}
|
||||
QuotaOperation::DeleteObject => true,
|
||||
};
|
||||
@@ -105,6 +110,7 @@ impl QuotaChecker {
|
||||
quota_limit: Some(quota_limit),
|
||||
operation_size,
|
||||
remaining,
|
||||
uses_durable_reservations,
|
||||
};
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
@@ -158,6 +164,26 @@ impl QuotaChecker {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn set_durable_quota_config_if_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
quota: BucketQuota,
|
||||
expected_incarnation_id: uuid::Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime, QuotaError> {
|
||||
let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig {
|
||||
reason: format!("Failed to serialize quota config: {}", e),
|
||||
})?;
|
||||
let start_time = Instant::now();
|
||||
let updated_at =
|
||||
crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
|
||||
.await
|
||||
.map_err(QuotaError::StorageError)?;
|
||||
|
||||
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
async fn set_quota_config_for_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
@@ -355,6 +381,7 @@ mod tests {
|
||||
quota_limit: None,
|
||||
operation_size: 1024,
|
||||
remaining: None,
|
||||
uses_durable_reservations: false,
|
||||
};
|
||||
|
||||
assert!(result.allowed);
|
||||
@@ -378,4 +405,13 @@ mod tests {
|
||||
let allowed = quota.check_operation_allowed(512, 1024);
|
||||
assert!(!allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
|
||||
let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
|
||||
let durable = BucketQuota::new(Some(5));
|
||||
|
||||
assert!(!legacy.check_operation_allowed(4, 2));
|
||||
assert!(durable.uses_durable_reservations());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,40 +13,100 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod checker;
|
||||
pub(crate) mod reservation;
|
||||
|
||||
use crate::error::Result;
|
||||
use rustfs_config::{
|
||||
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
|
||||
QUOTA_NOT_FOUND_ERROR_CODE,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||
use thiserror::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub enum QuotaType {
|
||||
/// Hard quota: reject immediately when exceeded
|
||||
/// Hard quota accounting.
|
||||
#[default]
|
||||
#[serde(alias = "HARD", alias = "hard")]
|
||||
Hard,
|
||||
}
|
||||
|
||||
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
|
||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
pub struct BucketQuota {
|
||||
#[serde(default)]
|
||||
pub quota: Option<u64>,
|
||||
/// Defaults to Hard when missing.
|
||||
#[serde(default)]
|
||||
pub quota_type: QuotaType,
|
||||
/// Optional durable reservation protocol. The wire format gives older
|
||||
/// nodes a zero hard quota so a mixed-version fleet fails closed.
|
||||
pub reservation_protocol: Option<u32>,
|
||||
/// Timestamp when this quota configuration was set (for audit purposes)
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
pub created_at: Option<OffsetDateTime>,
|
||||
/// Accept updated_at for compatibility; not used.
|
||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct BucketQuotaWire {
|
||||
#[serde(default)]
|
||||
quota: Option<u64>,
|
||||
#[serde(default)]
|
||||
quota_type: QuotaType,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_protocol: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_quota: Option<u64>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
created_at: Option<OffsetDateTime>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl Serialize for BucketQuota {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let durable = self.uses_durable_reservations();
|
||||
BucketQuotaWire {
|
||||
quota: if durable { Some(0) } else { self.quota },
|
||||
quota_type: self.quota_type.clone(),
|
||||
reservation_protocol: self.reservation_protocol,
|
||||
reservation_quota: if durable { self.quota } else { None },
|
||||
created_at: self.created_at,
|
||||
updated_at: self.updated_at,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BucketQuota {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||
Some(
|
||||
wire.reservation_quota
|
||||
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
|
||||
)
|
||||
} else {
|
||||
wire.quota
|
||||
};
|
||||
Ok(Self {
|
||||
quota,
|
||||
quota_type: wire.quota_type,
|
||||
reservation_protocol: wire.reservation_protocol,
|
||||
created_at: wire.created_at,
|
||||
updated_at: wire.updated_at,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketQuota {
|
||||
/// Serialize to JSON bytes. Same format as parse_all_configs.
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
@@ -63,6 +123,7 @@ impl BucketQuota {
|
||||
Self {
|
||||
quota,
|
||||
quota_type: QuotaType::Hard,
|
||||
reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
|
||||
created_at: Some(now),
|
||||
updated_at: None,
|
||||
}
|
||||
@@ -72,7 +133,19 @@ impl BucketQuota {
|
||||
self.quota
|
||||
}
|
||||
|
||||
pub fn uses_durable_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||
self.reservation_protocol
|
||||
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||
if operation_size == 0 {
|
||||
return true;
|
||||
}
|
||||
if let Some(quota_limit) = self.quota {
|
||||
current_usage.saturating_add(operation_size) <= quota_limit
|
||||
} else {
|
||||
@@ -94,6 +167,7 @@ pub struct QuotaCheckResult {
|
||||
pub quota_limit: Option<u64>,
|
||||
pub operation_size: u64,
|
||||
pub remaining: Option<u64>,
|
||||
pub uses_durable_reservations: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -119,6 +193,7 @@ pub enum QuotaError {
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub struct QuotaErrorResponse {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: String,
|
||||
@@ -134,6 +209,7 @@ pub struct QuotaErrorResponse {
|
||||
}
|
||||
|
||||
impl QuotaErrorResponse {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||
match quota_error {
|
||||
QuotaError::QuotaExceeded { .. } => Self {
|
||||
@@ -210,7 +286,59 @@ mod tests {
|
||||
let buf = q.marshal_msg().expect("marshal");
|
||||
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
|
||||
assert_eq!(q.quota, restored.quota);
|
||||
assert_eq!(q.quota_type, restored.quota_type);
|
||||
assert_eq!(restored.quota_type, QuotaType::Hard);
|
||||
assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clearing_quota_keeps_the_legacy_compatible_type() {
|
||||
let quota = BucketQuota::new(None);
|
||||
|
||||
assert_eq!(quota.quota_type, QuotaType::Hard);
|
||||
assert_eq!(quota.reservation_protocol, None);
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_quota_makes_legacy_nodes_fail_closed() {
|
||||
let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
|
||||
let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
|
||||
assert!(quota.uses_durable_reservations());
|
||||
assert_eq!(quota.quota, Some(2048));
|
||||
|
||||
#[derive(Deserialize)]
|
||||
enum LegacyQuotaType {
|
||||
Hard,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
#[allow(dead_code)]
|
||||
quota: Option<u64>,
|
||||
#[allow(dead_code)]
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
.expect("legacy readers should ignore the reservation protocol field");
|
||||
assert_eq!(legacy.quota, Some(0));
|
||||
assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||
let quota: BucketQuota =
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||
.expect("future protocol should remain parseable");
|
||||
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
assert!(quota.has_unsupported_reservation_protocol());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reservation_protocol_v1_requires_reservation_quota() {
|
||||
let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
|
||||
.expect_err("v1 without its authoritative quota must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("reservation_quota is required"));
|
||||
}
|
||||
|
||||
/// unmarshal accepts format without quota_type
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,7 @@ paths.
|
||||
| `datatypes.rs` | ECStore compatibility re-export for resync status enums. | Re-exports `rustfs-replication` contracts while downstream facade consumers migrate. |
|
||||
| `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. |
|
||||
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
|
||||
| `replication_proxy.rs` | Proxy-target selection for GET/HEAD/Tagging reads of objects not yet replicated locally (MinIO `getProxyTargets` parity: anti-loop, version-suspended, and no-config empty branches). | Uses replication config lookup, rule matching, and target clients through local boundaries. |
|
||||
| `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. |
|
||||
| `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. |
|
||||
| `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
|
||||
|
||||
@@ -29,6 +29,7 @@ mod replication_object_bridge;
|
||||
mod replication_object_config;
|
||||
mod replication_object_decision_boundary;
|
||||
pub(crate) mod replication_pool;
|
||||
mod replication_proxy;
|
||||
mod replication_queue_boundary;
|
||||
mod replication_resync_boundary;
|
||||
mod replication_resyncer;
|
||||
@@ -60,7 +61,7 @@ pub use replication_filemeta_boundary::{
|
||||
pub(crate) use replication_filemeta_boundary::{
|
||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
};
|
||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
@@ -74,6 +75,7 @@ pub use replication_pool::{
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_proxy::get_proxy_targets;
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
@@ -81,6 +83,6 @@ pub use replication_queue_boundary::{
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -37,6 +37,10 @@ impl ReplicationConfigStore {
|
||||
com::read_config_limited(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
|
||||
@@ -24,15 +24,27 @@ use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(
|
||||
config: &ReplicationLifecycleConfig,
|
||||
object_name: &str,
|
||||
@@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge {
|
||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: &ObjectToDelete,
|
||||
@@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge {
|
||||
check_replicate_delete(bucket, object, source, opts, None).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||
let pending_status = decision.pending_status();
|
||||
ReplicationState {
|
||||
|
||||
@@ -19,17 +19,33 @@ use time::OffsetDateTime;
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::msgp_decode;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) struct ReplicationMsgpCodec;
|
||||
|
||||
impl ReplicationMsgpCodec {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||
msgp_decode::read_msgp_ext8_time(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||
msgp_decode::skip_msgp_value(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||
msgp_decode::write_msgp_time(wr, time)
|
||||
}
|
||||
|
||||
@@ -77,6 +77,10 @@ impl ReplicationObjectBridge {
|
||||
load_delete_request_config_in(ctx, bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn delete_config_snapshot_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config(
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn load_delete_replication_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||
let mut tracker = DurableMrfBacklogTracker {
|
||||
available: true,
|
||||
@@ -663,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Some(guard),
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and decodes the on-disk MRF recovery file.
|
||||
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return None;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match decode_mrf_file(&data) {
|
||||
Ok(v) => Some(v),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(storage, &data).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays one MRF recovery entry by operation kind.
|
||||
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||
}
|
||||
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||
/// stale force-delete generations are skipped, and plain deletes are
|
||||
/// reconstructed as heal deletes.
|
||||
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let operation_id = entry.force_delete_id?;
|
||||
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
Some(ReplicationQueueAdmission::Skipped)
|
||||
} else {
|
||||
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
/// get_object_info here because the delete-marker or version may
|
||||
/// already be absent from the local store — that is expected.
|
||||
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The MRF entry does not persist the replication decision and the
|
||||
/// source object is gone, so re-derive the decision from the live
|
||||
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
/// it on the reconstructed delete. Without this the decision string
|
||||
/// is empty and the delete replicates to zero targets — a silent
|
||||
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
async fn resolve_mrf_delete_replicate_decision(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
versioned: bool,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicateDecision> {
|
||||
if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => None,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => Some(dsc),
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||
/// replication decision.
|
||||
fn reconstructed_heal_delete_info(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a metadata-kind MRF entry against the live source object.
|
||||
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||
/// admitted targets, carrying over the entry's retry count.
|
||||
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
roi
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
entries: &[MrfReplicateEntry],
|
||||
retry_entries: &[MrfReplicateEntry],
|
||||
) -> Vec<MrfReplicateEntry> {
|
||||
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.to_vec()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -712,6 +1078,10 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
|
||||
// MRF worker lifecycle
|
||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
mrf_stop_tx: Sender<()>,
|
||||
|
||||
// Worker size tracking
|
||||
@@ -940,6 +1310,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Resizes worker priority and counts
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn resize_worker_priority(
|
||||
&self,
|
||||
pri: ReplicationPriority,
|
||||
@@ -1180,6 +1554,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Queues an MRF save operation
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
@@ -1205,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let recovery_guard = match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries = match decode_mrf_file(&data) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(&storage, &data).await;
|
||||
return;
|
||||
}
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
@@ -1278,187 +1597,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let admission = match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let Some(operation_id) = entry.force_delete_id else {
|
||||
continue;
|
||||
};
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
ReplicationQueueAdmission::Skipped
|
||||
} else {
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
//
|
||||
// The MRF entry does not persist the replication decision and the
|
||||
// source object is gone, so re-derive the decision from the live
|
||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
// it on the reconstructed delete. Without this the decision string
|
||||
// is empty and the delete replicates to zero targets — a silent
|
||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => continue,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => dsc,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
||||
};
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Metadata => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
@@ -1468,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
@@ -1651,6 +1769,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling regular replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1664,6 +1786,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1678,6 +1804,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_mrf_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1691,6 +1821,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Delete resync metadata from replication resync state in memory
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||
let mut status_map = self.resyncer.status_map.write().await;
|
||||
status_map.remove(bucket);
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Proxy-target selection for reads of objects not yet replicated locally
|
||||
//! (MinIO `getProxyTargets`, bucket-replication.go).
|
||||
//!
|
||||
//! During the active-active replication lag window a GET/HEAD/Tagging request
|
||||
//! for an object the local site does not have yet may be served by proxying to
|
||||
//! a replication target. This module only *selects* the candidate targets; the
|
||||
//! request-path callers perform the remote calls and response translation.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_object_config::get_replication_config;
|
||||
use super::replication_storage_boundary::ObjectOptions;
|
||||
use super::replication_target_boundary::{ReplicationTargetStore, TargetClient};
|
||||
|
||||
/// Returns the replication-target clients eligible to serve a proxied read of
|
||||
/// `bucket/object`, in rule order. Mirrors MinIO's `getProxyTargets`:
|
||||
///
|
||||
/// - the `source-proxy-request` header family was present at all
|
||||
/// (`opts.proxy_request` / `opts.proxy_header_set`, MinIO `ProxyRequest` /
|
||||
/// `ProxyHeaderSet`) -> empty. "true" is the anti-loop marker of an
|
||||
/// already-proxied client read; "false" is what a peer's replication
|
||||
/// worker sends on convergence HEADs so the receiver answers locally —
|
||||
/// proxying that miss back would echo the source object and fake
|
||||
/// convergence, permanently skipping replication;
|
||||
/// - the bucket's versioning is suspended for the object -> empty;
|
||||
/// - no replication configuration / no matching rule -> empty;
|
||||
/// - otherwise every distinct target ARN whose rules match the object,
|
||||
/// resolved through the bucket target system, skipping targets that opted
|
||||
/// out of proxying (`disable_proxy`).
|
||||
pub async fn get_proxy_targets(bucket: &str, object: &str, opts: &ObjectOptions) -> Vec<Arc<TargetClient>> {
|
||||
if opts.proxy_request || opts.proxy_header_set {
|
||||
return Vec::new();
|
||||
}
|
||||
if opts.version_suspended {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let cfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(cfg)) => cfg,
|
||||
Ok(None) => return Vec::new(),
|
||||
Err(err) => {
|
||||
debug!(bucket, object, error = %err, "read proxy: failed to load replication config; not proxying");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let arns = cfg.filter_target_arns(&ObjectOpts {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut targets = Vec::with_capacity(arns.len());
|
||||
for arn in arns {
|
||||
let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
debug!(bucket, object, arn, "read proxy: no client for replication target ARN");
|
||||
continue;
|
||||
};
|
||||
if client.disable_proxy {
|
||||
continue;
|
||||
}
|
||||
targets.push(client);
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn opts() -> ObjectOptions {
|
||||
ObjectOptions::default()
|
||||
}
|
||||
|
||||
/// Anti-loop: a request that was already proxied by a peer must never be
|
||||
/// proxied onward, regardless of replication configuration.
|
||||
#[tokio::test]
|
||||
async fn proxy_request_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_request: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// MinIO `ProxyHeaderSet` parity: the header family being present at all
|
||||
/// disables proxying, even with the value "false" — that is what a
|
||||
/// peer's replication worker sends on convergence HEADs.
|
||||
#[tokio::test]
|
||||
async fn proxy_header_set_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_header_set: true,
|
||||
proxy_request: false,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// Suspended versioning disables proxying (MinIO parity): the local null
|
||||
/// version is authoritative and a remote read could resurrect data.
|
||||
#[tokio::test]
|
||||
async fn version_suspended_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// A bucket without replication configuration has nothing to proxy to.
|
||||
/// (No metadata system is running in unit tests, so the config lookup
|
||||
/// resolves to "no configuration" — the same empty-result contract.)
|
||||
#[tokio::test]
|
||||
async fn missing_replication_config_yields_no_targets() {
|
||||
let targets = get_proxy_targets("bucket-without-replication", "object", &opts()).await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -15,17 +15,41 @@
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
use super::replication_filemeta_boundary::MrfReplicateEntry;
|
||||
|
||||
/// Kept test-only: the runtime consumer was the worker HEAD's fake proxy
|
||||
/// counting (removed in backlog#1675 P1-5); the resyncer tests still pin the
|
||||
/// classifier's semantics for the real client read-proxy failure accounting.
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::should_count_head_proxy_failure;
|
||||
pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus};
|
||||
pub(crate) use rustfs_replication::{
|
||||
is_version_id_mismatch, resync_state_accepts_update, sanitize_resync_error_detail, should_auto_resume_resync,
|
||||
should_count_head_proxy_failure,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||
|
||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -340,6 +340,10 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Site replication update replica statistics
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn sr_update_replica_stat(&self, size: i64) {
|
||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -704,6 +708,12 @@ impl ReplicationStats {
|
||||
} else {
|
||||
BucketReplicationStats::new()
|
||||
};
|
||||
// Stamp the serializable failure windows from the live samples: the
|
||||
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
||||
// is what cluster aggregation and the metrics endpoints see.
|
||||
for stat in replication_stats.stats.values_mut() {
|
||||
stat.fail_stats.refresh_windows();
|
||||
}
|
||||
let uptime = if cache.contains_key(bucket) {
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
@@ -1151,6 +1161,31 @@ mod tests {
|
||||
assert!(all.contains_key("proxy-only-bucket"));
|
||||
}
|
||||
|
||||
/// Pins the read-proxy metric contract (backlog#1675 P1-5): the API
|
||||
/// strings the GET/HEAD/Tagging proxy paths record map onto the
|
||||
/// get/head/tagging totals, and only unexpected failures raise the
|
||||
/// failed counters.
|
||||
#[tokio::test]
|
||||
async fn test_proxy_stats_map_read_proxy_apis_to_totals() {
|
||||
let stats = ReplicationStats::new();
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", true).await;
|
||||
stats.inc_proxy("proxy-bucket", "HeadObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "PutObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "DeleteObjectTagging", true).await;
|
||||
|
||||
let metric = stats.get_proxy_stats("proxy-bucket").await;
|
||||
assert_eq!(metric.get_total, 2);
|
||||
assert_eq!(metric.get_failed, 1);
|
||||
assert_eq!(metric.head_total, 1);
|
||||
assert_eq!(metric.head_failed, 0);
|
||||
assert_eq!(metric.get_tag_total, 1);
|
||||
assert_eq!(metric.put_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_failed, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_calculate_bucket_replication_stats_merges_resync_metrics() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||
SRMetricsSummary, XferStats,
|
||||
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
||||
};
|
||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
||||
// can project the internal stats onto the minio-go response shapes through
|
||||
// the storage_api facade chain.
|
||||
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
|
||||
@@ -27,14 +27,17 @@ use rustfs_utils::http::{
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
||||
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
||||
ssec_replication_transport_header,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, SsecPassthroughCapability, TargetClient,
|
||||
resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
@@ -63,6 +66,8 @@ static STANDARD_HEADERS: &[&str] = &[
|
||||
];
|
||||
|
||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||
pub(crate) const ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED: &str = "replication target does not support SSE-C passthrough: the replica would lose its decryption material \
|
||||
(run ?replication-check to re-probe)";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplicationSourceEncryption {
|
||||
@@ -119,10 +124,79 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
||||
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
||||
let key = key.as_bytes();
|
||||
key.len() == prefix.len() + suffix.len()
|
||||
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
||||
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
||||
}
|
||||
|
||||
[
|
||||
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| {
|
||||
["x-rustfs-", "x-minio-"]
|
||||
.iter()
|
||||
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// Fail-closed decision for an SSE-C passthrough replication attempt, derived
|
||||
/// from the target's cached [`SsecPassthroughCapability`]. Pure so the policy
|
||||
/// can migrate with the worker (M2) without dragging the cache along; the
|
||||
/// caller computes `expired` from the cache record's age (see
|
||||
/// `SSEC_PASSTHROUGH_CAPABILITY_TTL`).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum SsecPassthroughGate {
|
||||
/// Not an SSE-C object, or the target has a fresh proof that it preserves
|
||||
/// the passthrough transport headers: replicate without a HEAD-back audit.
|
||||
Proceed,
|
||||
/// No usable verdict — first SSE-C attempt since the target was (re)built,
|
||||
/// or the recorded verdict (in either direction) aged out: PUT, then HEAD
|
||||
/// the replica back and require SSE-C evidence before reporting COMPLETED.
|
||||
ProceedWithAudit,
|
||||
/// The target was recently proven to drop the passthrough headers: do not
|
||||
/// send the PUT, report FAILED (the object stays on the normal MRF retry
|
||||
/// channel and re-audits once the verdict expires).
|
||||
FailClosed,
|
||||
}
|
||||
|
||||
pub(crate) fn ssec_passthrough_gate(ssec: bool, capability: SsecPassthroughCapability, expired: bool) -> SsecPassthroughGate {
|
||||
if !ssec {
|
||||
return SsecPassthroughGate::Proceed;
|
||||
}
|
||||
// An expired verdict — Supported or Unsupported — must be re-earned: a
|
||||
// stale Unsupported would otherwise stick forever after a target upgrade,
|
||||
// and a stale Supported would fail open after a backend swap behind the
|
||||
// same endpoint.
|
||||
if expired {
|
||||
return SsecPassthroughGate::ProceedWithAudit;
|
||||
}
|
||||
match capability {
|
||||
SsecPassthroughCapability::Supported => SsecPassthroughGate::Proceed,
|
||||
SsecPassthroughCapability::Unknown => SsecPassthroughGate::ProceedWithAudit,
|
||||
SsecPassthroughCapability::Unsupported => SsecPassthroughGate::FailClosed,
|
||||
}
|
||||
}
|
||||
|
||||
/// True when a replication-check HEAD of the replica proves the SSE-C
|
||||
/// material survived passthrough: a RustFS target restores the transport
|
||||
/// headers into the stored SSE-C keys and its HEAD echoes
|
||||
/// `x-amz-server-side-encryption-customer-algorithm` (the replication-check
|
||||
/// exemption skips key validation but not the metadata echo). A target that
|
||||
/// dropped the headers stored a plain object and echoes nothing.
|
||||
pub(crate) fn ssec_passthrough_evidence_present(head: &HeadObjectOutput) -> bool {
|
||||
head.sse_customer_algorithm.as_deref().is_some_and(|algo| !algo.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) struct ReplicationTargetStore;
|
||||
|
||||
impl ReplicationTargetStore {
|
||||
@@ -142,6 +216,17 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||
BucketTargetSys::get()
|
||||
.record_ssec_passthrough_capability(arn, capability)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -176,6 +261,11 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_legacy_source_replication_timestamp_key(key) {
|
||||
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_internal_key(key) || is_standard_header(key) {
|
||||
continue;
|
||||
}
|
||||
@@ -259,15 +349,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
|
||||
if !tags.is_empty() {
|
||||
put_options.user_tags = tags;
|
||||
put_options.internal.tagging_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Load the stored tagging timestamp independently of whether any tags
|
||||
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||
// With no stored key, fall back to mod_time only while tags exist
|
||||
// (MinIO parity); a tagless object without the key was never tagged and
|
||||
// keeps the epoch default (no header).
|
||||
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||
{
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else if !put_options.user_tags.is_empty() {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
let metadata = &*object_info.user_defined;
|
||||
|
||||
@@ -283,13 +381,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
put_options.cache_control = cache_control.to_string();
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
if !retain_until_date.is_empty() {
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
}
|
||||
put_options.internal.retention_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
@@ -694,6 +794,110 @@ mod tests {
|
||||
assert!(options.internal.replication_request);
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||
/// still be loaded (and therefore sent) so the replica can order the
|
||||
/// deletion against concurrent tag edits.
|
||||
#[test]
|
||||
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
assert!(options.user_tags.is_empty());
|
||||
assert_eq!(
|
||||
options.internal.tagging_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||
"the stored tagging timestamp must load independently of remaining tags"
|
||||
);
|
||||
|
||||
// A tagless object without the stored key was never tagged: the epoch
|
||||
// default keeps the header unsent.
|
||||
let untagged = ObjectInfo {
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
||||
let legacy_keys = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(
|
||||
legacy_keys
|
||||
.iter()
|
||||
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
for legacy_key in legacy_keys {
|
||||
assert!(!options.user_metadata.contains_key(legacy_key));
|
||||
assert_eq!(
|
||||
options
|
||||
.user_metadata
|
||||
.get(&format!("x-amz-meta-{legacy_key}"))
|
||||
.map(String::as_str),
|
||||
Some("2099-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
||||
let mut metadata = HashMap::from([
|
||||
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
||||
]);
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
||||
|
||||
assert!(options.mode.is_none());
|
||||
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(
|
||||
options.internal.retention_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
||||
);
|
||||
let headers = options.header();
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
||||
Some("2026-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||
@@ -756,6 +960,71 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// N2 fail-closed policy: SSE-C replication may only proceed silently
|
||||
/// against a target with a FRESH proof that it preserves the passthrough
|
||||
/// transport headers. Unknown targets must be audited; freshly-flagged
|
||||
/// dropping targets must never receive the PUT; an expired verdict in
|
||||
/// EITHER direction must be re-earned through the audit — a sticky
|
||||
/// Unsupported would outlive a target upgrade, and a sticky Supported
|
||||
/// would fail open after a backend swap behind the same endpoint.
|
||||
#[test]
|
||||
fn ssec_passthrough_gate_is_fail_closed_and_ttl_bounded() {
|
||||
for capability in [
|
||||
SsecPassthroughCapability::Unknown,
|
||||
SsecPassthroughCapability::Supported,
|
||||
SsecPassthroughCapability::Unsupported,
|
||||
] {
|
||||
for expired in [false, true] {
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(false, capability, expired),
|
||||
SsecPassthroughGate::Proceed,
|
||||
"non-SSE-C objects must never be gated on the passthrough capability"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(true, SsecPassthroughCapability::Supported, false),
|
||||
SsecPassthroughGate::Proceed
|
||||
);
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(true, SsecPassthroughCapability::Unknown, false),
|
||||
SsecPassthroughGate::ProceedWithAudit
|
||||
);
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(true, SsecPassthroughCapability::Unsupported, false),
|
||||
SsecPassthroughGate::FailClosed
|
||||
);
|
||||
// Expiry flips both directions back to the audit.
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(true, SsecPassthroughCapability::Unsupported, true),
|
||||
SsecPassthroughGate::ProceedWithAudit,
|
||||
"an expired Unsupported verdict must allow a re-audit (upgraded target recovers without operator action)"
|
||||
);
|
||||
assert_eq!(
|
||||
ssec_passthrough_gate(true, SsecPassthroughCapability::Supported, true),
|
||||
SsecPassthroughGate::ProceedWithAudit,
|
||||
"an expired Supported verdict must be re-proven (backend swap behind the same endpoint must not fail open)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssec_passthrough_evidence_requires_customer_algorithm_echo() {
|
||||
let with_evidence = HeadObjectOutput::builder().sse_customer_algorithm("AES256").build();
|
||||
assert!(ssec_passthrough_evidence_present(&with_evidence));
|
||||
|
||||
let empty_algorithm = HeadObjectOutput::builder().sse_customer_algorithm("").build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&empty_algorithm),
|
||||
"an empty echo is not evidence of preserved SSE-C material"
|
||||
);
|
||||
|
||||
let without_evidence = HeadObjectOutput::builder().e_tag("\"abc\"").content_length(8).build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&without_evidence),
|
||||
"a plain HEAD response must classify the target as having dropped the material"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_adds_ssec_checksum_metadata() {
|
||||
let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||
|
||||
@@ -40,7 +40,14 @@ impl ARN {
|
||||
|
||||
impl Display for ARN {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
// The `minio` partition is deliberate: madmin-go's ParseARN
|
||||
// hard-rejects any other partition, so native mc/madmin tooling can
|
||||
// only decode remote-target ARNs minted in this form (backlog#1675
|
||||
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
||||
// readable via the FromStr whitelist below; runtime matching between
|
||||
// targets and replication rules is by full-string equality, so mixed
|
||||
// partitions coexist safely.
|
||||
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +55,12 @@ impl FromStr for ARN {
|
||||
type Err = std::io::Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
if !s.starts_with("arn:rustfs:") {
|
||||
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
||||
// from_str(...).unwrap_or_default()` below never fails, so this is
|
||||
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
||||
// the legacy partition and must stay accepted forever (persisted
|
||||
// bucket-targets.json / replication configs from older releases).
|
||||
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||
}
|
||||
|
||||
@@ -101,14 +113,50 @@ mod tests {
|
||||
}
|
||||
|
||||
/// RustFS commonly generates ARNs with an empty region:
|
||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
||||
#[test]
|
||||
fn from_str_handles_empty_region_segment() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
||||
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
||||
/// native mc/madmin tooling cannot decode remote-target listings.
|
||||
#[test]
|
||||
fn display_emits_minio_partition() {
|
||||
let arn = ARN::new(
|
||||
BucketTargetType::ReplicationService,
|
||||
"depl-123".to_string(),
|
||||
String::new(),
|
||||
"bucket-a".to_string(),
|
||||
);
|
||||
|
||||
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
||||
}
|
||||
|
||||
/// Persisted bucket-targets.json files from older RustFS releases carry
|
||||
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
||||
#[test]
|
||||
fn from_str_accepts_legacy_rustfs_partition() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "us-east-1");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
||||
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
||||
/// would otherwise parse as `type=None`.
|
||||
#[test]
|
||||
fn from_str_rejects_unknown_partition() {
|
||||
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
||||
assert!(ARN::from_str("not-an-arn").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,10 @@ impl fmt::Debug for Credentials {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub enum ServiceType {
|
||||
#[default]
|
||||
Replication,
|
||||
|
||||
@@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
||||
if object_name.len() > 1024 {
|
||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
||||
}
|
||||
if !object_name.is_ascii() {
|
||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
||||
if object_name.trim().is_empty() {
|
||||
return Err(Error::other("Object name cannot be empty"));
|
||||
}
|
||||
check_valid_object_name_prefix(object_name)
|
||||
}
|
||||
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
@@ -100,6 +83,10 @@ where
|
||||
Ok(ans)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
@@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||
/// accept the full S3 key character set.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||
|
||||
/// Client-facing reason for path segments Windows can store but not address
|
||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||
|
||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||
@@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||
object.split(['/', '\\']).any(|segment| {
|
||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||
|
||||
@@ -90,6 +90,10 @@ impl BucketVersioningSys {
|
||||
/// caller's own instance context so a second in-process store never
|
||||
/// answers with the first instance's versioning state; falls back to the
|
||||
/// ambient system when the instance cell is not initialized.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||
return Ok(VersioningConfiguration::default());
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
use std::{
|
||||
@@ -655,6 +656,7 @@ async fn list_path_raw_inner(
|
||||
errs.push(None);
|
||||
}
|
||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -676,6 +678,21 @@ async fn list_path_raw_inner(
|
||||
let mut has_err = 0;
|
||||
let mut agree = 0;
|
||||
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cancel = &revjob_rx;
|
||||
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
||||
});
|
||||
for (i, outcome) in join_all(concurrent_peeks).await {
|
||||
peek_outcomes[i] = Some(outcome);
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
has_err += 1;
|
||||
@@ -685,7 +702,10 @@ async fn list_path_raw_inner(
|
||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||
entry
|
||||
} else {
|
||||
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
||||
let Some(outcome) = peek_outcomes[i].take() else {
|
||||
return Err(DiskError::Unexpected);
|
||||
};
|
||||
match outcome {
|
||||
PeekOutcome::Ready(res) => {
|
||||
if let Some(entry) = res {
|
||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||
@@ -1295,6 +1315,36 @@ mod tests {
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
||||
let peek_timeout = Duration::from_millis(20);
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None, None, None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
],
|
||||
peek_timeout: Some(peek_timeout),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("all stalled readers should fail the listing");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(
|
||||
started.elapsed(),
|
||||
peek_timeout,
|
||||
"reader deadlines must overlap instead of accumulating once per disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||
let entry = MetaCacheEntry {
|
||||
|
||||
@@ -229,17 +229,6 @@ pub fn http_resp_to_error_response(
|
||||
err_resp
|
||||
}
|
||||
|
||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
||||
.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||
@@ -295,16 +284,6 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
||||
message: message.to_string(),
|
||||
request_id: "rustfs".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -135,6 +135,10 @@ impl Object {
|
||||
Self { ..Default::default() }
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||
let _ = request.did_offset_change;
|
||||
let _ = request.offset;
|
||||
@@ -150,12 +154,20 @@ impl Object {
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||
self.curr_offset += bytes_read;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||
let mut read_req = GetRequest {
|
||||
is_read_op: true,
|
||||
@@ -180,6 +192,10 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let _ = self.do_get_request(&GetRequest {
|
||||
@@ -192,6 +208,10 @@ impl Object {
|
||||
Ok(self.object_info.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||
self.curr_offset = offset;
|
||||
|
||||
@@ -219,6 +239,10 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let seek_req = GetRequest {
|
||||
@@ -253,6 +277,10 @@ impl Object {
|
||||
Ok(self.curr_offset)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||
self.is_closed = true;
|
||||
Ok(())
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::client::{
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
||||
credentials::SignatureType,
|
||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||
|
||||
@@ -30,10 +30,6 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||
let unknown_size;
|
||||
let mut object_size = object_size;
|
||||
|
||||
@@ -81,18 +81,6 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UploadedPartRes {
|
||||
pub error: std::io::Error,
|
||||
pub part_num: i64,
|
||||
pub size: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
pub struct UploadPartReq {
|
||||
pub part_num: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object_multipart_stream(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -29,10 +29,6 @@ use crate::client::utils::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
pub struct ListAllMyBucketsResult {
|
||||
pub owner: Owner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CommonPrefix {
|
||||
pub prefix: String,
|
||||
@@ -89,6 +85,10 @@ pub struct ListVersionsResult {
|
||||
pub next_version_id_marker: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListBucketResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
contents: Vec<transition_api::ObjectInfo>,
|
||||
@@ -102,6 +102,10 @@ pub struct ListBucketResult {
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListMultipartUploadsResult {
|
||||
bucket: String,
|
||||
key_marker: String,
|
||||
@@ -117,16 +121,15 @@ pub struct ListMultipartUploadsResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct Initiator {
|
||||
id: String,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
pub struct CopyObjectResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectPart {
|
||||
pub etag: String,
|
||||
@@ -260,6 +263,7 @@ pub struct CompletePart {
|
||||
}
|
||||
|
||||
impl CompletePart {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||
match t {
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
@@ -284,11 +288,6 @@ impl CompletePart {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CopyObjectPartResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
#[serde(rename = "CompleteMultipartUpload")]
|
||||
pub struct CompleteMultipartUpload {
|
||||
@@ -357,10 +356,10 @@ impl CompleteMultipartUpload {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CreateBucketConfiguration {
|
||||
pub location: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
||||
)]
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteObject {
|
||||
//api has
|
||||
@@ -368,21 +367,6 @@ pub struct DeleteObject {
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
pub struct DeletedObject {
|
||||
//s3s has
|
||||
pub key: String,
|
||||
pub version_id: String,
|
||||
pub deletemarker: bool,
|
||||
pub deletemarker_version_id: String,
|
||||
}
|
||||
|
||||
pub struct NonDeletedObject {
|
||||
pub key: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteMultiObjects {
|
||||
pub quiet: bool,
|
||||
@@ -402,6 +386,7 @@ impl DeleteMultiObjects {
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WireDeleteObject {
|
||||
@@ -436,8 +421,3 @@ impl DeleteMultiObjects {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DeleteMultiObjectsResult {
|
||||
pub deleted_objects: Vec<DeletedObject>,
|
||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
||||
}
|
||||
|
||||
@@ -365,6 +365,10 @@ mod tests {
|
||||
pub struct Checksum {
|
||||
checksum_type: ChecksumMode,
|
||||
r: Vec<u8>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
||||
)]
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,5 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||
|
||||
pub const TOTAL_WORKERS: i64 = 4;
|
||||
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
@@ -67,6 +67,10 @@ impl<P: Provider + Default> Credentials<P> {
|
||||
Ok(self.creds.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn expire(&mut self) {
|
||||
self.force_refresh = true;
|
||||
}
|
||||
@@ -133,6 +137,10 @@ impl Provider for Static {
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct STSError {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub r#type: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
@@ -141,6 +149,10 @@ pub struct STSError {
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub struct ErrorResponse {
|
||||
pub sts_error: STSError,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub request_id: String,
|
||||
}
|
||||
|
||||
@@ -158,22 +170,3 @@ impl ErrorResponse {
|
||||
return self.sts_error.message.clone();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
match std::str::from_utf8(body) {
|
||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
let body = body_reader.to_vec();
|
||||
let parsed = xml_decoder(&body)?;
|
||||
Ok((body, parsed))
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_error_response;
|
||||
|
||||
@@ -77,39 +77,6 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
|
||||
})
|
||||
}
|
||||
|
||||
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
||||
let mut skip_length: i64 = 0;
|
||||
let mut cumulative_actual_size: i64 = 0;
|
||||
let mut first_part_idx: i64 = 0;
|
||||
let mut compressed_offset: i64 = 0;
|
||||
let mut part_skip: i64 = 0;
|
||||
let mut decrypt_skip: i64 = 0;
|
||||
let mut seq_num: u64 = 0;
|
||||
for (i, part) in oi.parts.iter().enumerate() {
|
||||
cumulative_actual_size += part.actual_size as i64;
|
||||
if cumulative_actual_size <= offset {
|
||||
compressed_offset += part.size as i64;
|
||||
} else {
|
||||
first_part_idx = i as i64;
|
||||
skip_length = cumulative_actual_size - part.actual_size as i64;
|
||||
break;
|
||||
}
|
||||
}
|
||||
skip_length = offset - skip_length;
|
||||
|
||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
||||
if skip_length > 0
|
||||
&& parts.len() > first_part_idx as usize
|
||||
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
||||
{
|
||||
let _ = part_skip;
|
||||
let _ = decrypt_skip;
|
||||
let _ = seq_num;
|
||||
}
|
||||
|
||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
||||
}
|
||||
|
||||
pub fn new_getobjectreader<'a>(
|
||||
rs: &Option<HTTPRangeSpec>,
|
||||
oi: &'a ObjectInfo,
|
||||
|
||||
@@ -23,6 +23,7 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
|
||||
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||
pub(crate) enum BucketVersioningState {
|
||||
Unknown,
|
||||
Disabled,
|
||||
@@ -47,6 +48,7 @@ impl RemoteVersion {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||
match self {
|
||||
Self::Unknown => Err(Error::new(
|
||||
|
||||
@@ -101,6 +101,10 @@ where
|
||||
|
||||
const C_UNKNOWN: i32 = -1;
|
||||
const C_OFFLINE: i32 = 0;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reachable only from the unused transition client methods below (backlog#1823)"
|
||||
)]
|
||||
const C_ONLINE: i32 = 1;
|
||||
|
||||
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
||||
@@ -320,6 +324,10 @@ impl TransitionClient {
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn endpoint_url(&self) -> Url {
|
||||
self.endpoint_url.clone()
|
||||
}
|
||||
@@ -348,12 +356,20 @@ impl TransitionClient {
|
||||
.to_string())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn trace_errors_only_off(&self) {
|
||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||
*trace_errors_only = false;
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn trace_off(&self) {
|
||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||
*is_trace_enabled = false;
|
||||
@@ -363,12 +379,20 @@ impl TransitionClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||
*endpoint = accelerate_endpoint.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||
*dual_stack = enabled;
|
||||
@@ -398,10 +422,18 @@ impl TransitionClient {
|
||||
(hash_algos, hash_sums)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn is_online(&self) -> bool {
|
||||
!self.is_offline()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn mark_offline(&self) {
|
||||
self.health_status
|
||||
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
||||
@@ -411,10 +443,18 @@ impl TransitionClient {
|
||||
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn health_check(hc_duration: Duration) {
|
||||
let _ = hc_duration;
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
||||
let mut resp_trace: Vec<u8>;
|
||||
|
||||
@@ -1102,6 +1142,7 @@ impl Default for ObjectInfo {
|
||||
}
|
||||
|
||||
impl ObjectInfo {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn remote_version(
|
||||
&self,
|
||||
capabilities: ProviderVersionCapabilities,
|
||||
|
||||
@@ -48,10 +48,6 @@ lazy_static! {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
||||
SUPPORTED_QUERY_VALUES[qs_key]
|
||||
}
|
||||
|
||||
pub fn is_storageclass_header(header_key: &str) -> bool {
|
||||
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: cluster/RPC migration leaves transport capabilities staged for upcoming owners.
|
||||
#![allow(dead_code)]
|
||||
|
||||
mod control_plane;
|
||||
pub(crate) mod rpc;
|
||||
|
||||
@@ -256,6 +256,7 @@ impl<S> ReplayScopeChannel<S> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "replay-state probe asserted by this file's tests (backlog#1823)")]
|
||||
fn peer_replay_state(audience: &str) -> PeerReplayState {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
|
||||
@@ -31,7 +31,7 @@ use rustfs_config::{
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
|
||||
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
};
|
||||
use rustfs_rio::{HttpReader, HttpWriter};
|
||||
use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -43,6 +43,10 @@ use tokio::io::{AsyncReadExt, AsyncWrite};
|
||||
use tokio::sync::OnceCell;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live in the cfg(not(test)) half of build_internode_data_transport_from_env (backlog#1823)"
|
||||
)]
|
||||
static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
|
||||
|
||||
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
|
||||
@@ -134,6 +138,10 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "capability-negotiation seam; constructed only by transport test doubles (backlog#1823)"
|
||||
)]
|
||||
pub struct InternodeDataTransportCapabilities {
|
||||
/// Backend can open a streaming remote disk reader.
|
||||
pub streaming_read: bool,
|
||||
@@ -150,6 +158,10 @@ pub struct InternodeDataTransportCapabilities {
|
||||
}
|
||||
|
||||
impl InternodeDataTransportCapabilities {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "capability-negotiation seam; used by transport test doubles (backlog#1823)"
|
||||
)]
|
||||
pub const fn tcp_http() -> Self {
|
||||
Self {
|
||||
streaming_read: true,
|
||||
@@ -221,6 +233,11 @@ pub struct NsScannerCapabilityRequest {
|
||||
#[async_trait]
|
||||
pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
|
||||
/// Opens an owned-chunk stream when this transport can retain receive-buffer
|
||||
/// ownership. `None` preserves the established `open_read` fallback.
|
||||
async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
|
||||
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
|
||||
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
|
||||
@@ -229,7 +246,12 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
|
||||
Err(Error::MethodNotAllowed)
|
||||
}
|
||||
// Interface facet nobody calls yet: every transport implements both, but no
|
||||
// caller negotiates on them. Kept for the internode transport split
|
||||
// (backlog#1350); deleting them would delete the seam and six impls.
|
||||
#[allow(dead_code, reason = "unused capability-negotiation facet (backlog#1823)")]
|
||||
fn name(&self) -> &'static str;
|
||||
#[allow(dead_code, reason = "unused capability-negotiation facet (backlog#1823)")]
|
||||
fn capabilities(&self) -> InternodeDataTransportCapabilities;
|
||||
}
|
||||
|
||||
@@ -247,6 +269,15 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
))
|
||||
}
|
||||
|
||||
async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
let url = build_read_file_stream_url(&request);
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
Ok(Some(Box::new(
|
||||
HttpChunkReader::new_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout).await?,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let nonce = server_epoch.map(|_| Uuid::new_v4());
|
||||
@@ -656,6 +687,10 @@ fn build_internode_data_transport_result(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live in the cfg(test) half of build_internode_data_transport_from_env, which bypasses the process static (backlog#1823)"
|
||||
)]
|
||||
pub fn build_internode_data_transport(configured_transport: Option<&str>) -> Result<Arc<dyn InternodeDataTransport>> {
|
||||
build_internode_data_transport_result(configured_transport).map_err(Error::other)
|
||||
}
|
||||
|
||||
@@ -248,6 +248,16 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8])
|
||||
Ok(server_epoch)
|
||||
}
|
||||
|
||||
fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> {
|
||||
let version = result
|
||||
.get(..4)
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.map(u32::from_be_bytes)
|
||||
.ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?;
|
||||
let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?;
|
||||
Ok((version, epoch))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -1288,6 +1298,16 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
|
||||
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
|
||||
probe.extend_from_slice(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -2738,6 +2758,24 @@ mod tests {
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_pool_fence_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"),
|
||||
(1, epoch)
|
||||
);
|
||||
for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] {
|
||||
assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err());
|
||||
}
|
||||
assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err());
|
||||
let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
|
||||
@@ -854,7 +854,6 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RemotePeerS3Client {
|
||||
pub node: Option<Node>,
|
||||
pub pools: Option<Vec<usize>>,
|
||||
addr: String,
|
||||
/// Health tracker for connection monitoring
|
||||
@@ -886,7 +885,6 @@ impl RemotePeerS3Client {
|
||||
pub fn new(node: Option<Node>, pools: Option<Vec<usize>>) -> Self {
|
||||
let addr = node.as_ref().map(|v| v.url.to_string()).unwrap_or_default();
|
||||
let client = Self {
|
||||
node,
|
||||
pools,
|
||||
addr,
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
@@ -905,10 +903,6 @@ impl RemotePeerS3Client {
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
||||
}
|
||||
|
||||
pub fn get_addr(&self) -> String {
|
||||
self.addr.clone()
|
||||
}
|
||||
|
||||
/// Start health monitoring for the remote peer
|
||||
fn start_health_monitoring(&self) {
|
||||
let health = Arc::clone(&self.health);
|
||||
@@ -1208,6 +1202,10 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "local bucket-heal path reached only by this file's tests (backlog#1823)"
|
||||
)]
|
||||
pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
let disks = clone_drives().await;
|
||||
heal_bucket_local_on_disks(bucket, opts, disks).await
|
||||
@@ -1404,6 +1402,10 @@ pub(crate) async fn heal_bucket_local_on_disks(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only through heal_bucket_local, which only tests call (backlog#1823)"
|
||||
)]
|
||||
async fn clone_drives() -> Vec<Option<DiskStore>> {
|
||||
runtime_sources::local_disk_entries().await
|
||||
}
|
||||
@@ -1585,15 +1587,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
|
||||
let node = Node {
|
||||
url: url::Url::parse(addr).expect("test peer URL should parse"),
|
||||
pools: vec![0],
|
||||
is_local: false,
|
||||
grid_host: addr.to_string(),
|
||||
};
|
||||
|
||||
RemotePeerS3Client {
|
||||
node: Some(node),
|
||||
pools: Some(vec![0]),
|
||||
addr: addr.to_string(),
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
|
||||
@@ -522,6 +522,33 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_read_chunks_with_retry(&self, request: ReadStreamRequest) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
let mut attempt = 1;
|
||||
let mut last_retry_classification = None;
|
||||
loop {
|
||||
match self.data_transport.open_read_chunks(request.clone()).await {
|
||||
Ok(reader) => {
|
||||
if attempt > 1
|
||||
&& let Some(classification) = last_retry_classification
|
||||
{
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_open_read_retry_success(classification);
|
||||
}
|
||||
return Ok(reader);
|
||||
}
|
||||
Err(err) if attempt < REMOTE_DISK_OPEN_READ_MAX_ATTEMPTS && Self::is_retryable_open_read_error(&err) => {
|
||||
if let Some(classification) = err.internode_http_error_kind() {
|
||||
let classification = classification.metric_label();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_open_read_retry(classification);
|
||||
last_retry_classification = Some(classification);
|
||||
}
|
||||
tokio::time::sleep(REMOTE_DISK_OPEN_READ_RETRY_BACKOFF).await;
|
||||
attempt += 1;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
|
||||
self.health.record_capacity_probe(total, used, free);
|
||||
}
|
||||
@@ -846,31 +873,49 @@ impl RemoteDisk {
|
||||
/// default to 1 (see [`internode_idempotent_read_retries`]). MUST NOT be used for write/lock
|
||||
/// RPCs — those must never auto-retry (quorum/idempotency safety). The `operation` closure is
|
||||
/// re-invoked per attempt, so it must be `Fn` (rebuild the request from borrowed inputs, do not
|
||||
/// move captured state out).
|
||||
/// move captured state out). Attempts and backoff share one total timeout budget.
|
||||
async fn execute_read_with_retry<T, F, Fut>(&self, op: &'static str, operation: F, timeout_duration: Duration) -> Result<T>
|
||||
where
|
||||
F: Fn() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
let deadline = (!timeout_duration.is_zero()).then(|| {
|
||||
time::Instant::now()
|
||||
.checked_add(timeout_duration)
|
||||
.unwrap_or_else(|| time::sleep(timeout_duration).deadline())
|
||||
});
|
||||
let max_retries = internode_idempotent_read_retries();
|
||||
let mut attempt = 0usize;
|
||||
loop {
|
||||
// Only the final attempt marks the disk faulty / evicts the channel. Earlier retries
|
||||
// ignore the failure, so a transient error cannot flip the disk into a faulty
|
||||
// short-circuit (which would defeat the retry) or over-count failures.
|
||||
let attempt_timeout = deadline
|
||||
.map(|deadline| deadline.saturating_duration_since(time::Instant::now()))
|
||||
.unwrap_or(Duration::ZERO);
|
||||
if deadline.is_some() && attempt_timeout.is_zero() {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
|
||||
let health_action = if attempt >= max_retries {
|
||||
FailureHealthAction::MarkFailure
|
||||
} else {
|
||||
FailureHealthAction::IgnoreFailure
|
||||
};
|
||||
match self
|
||||
.execute_with_timeout_for_op_and_health_action(op, &operation, timeout_duration, health_action)
|
||||
.execute_with_timeout_for_op_and_health_action(op, &operation, attempt_timeout, health_action)
|
||||
.await
|
||||
{
|
||||
Err(err) if attempt < max_retries && is_network_like_disk_error(&err) => {
|
||||
if matches!(err, DiskError::Timeout) && deadline.is_some_and(|deadline| time::Instant::now() >= deadline) {
|
||||
self.mark_faulty("read_operation_deadline");
|
||||
return Err(err);
|
||||
}
|
||||
attempt += 1;
|
||||
let backoff = REMOTE_DISK_READ_RETRY_BASE_BACKOFF
|
||||
.saturating_mul(1u32 << u32::try_from(attempt - 1).unwrap_or(4).min(4));
|
||||
if deadline.is_some_and(|deadline| deadline.saturating_duration_since(time::Instant::now()) <= backoff) {
|
||||
attempt = max_retries;
|
||||
continue;
|
||||
}
|
||||
debug!(
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
@@ -878,7 +923,17 @@ impl RemoteDisk {
|
||||
attempt,
|
||||
"retrying idempotent read-only RPC after transient network error"
|
||||
);
|
||||
tokio::time::sleep(backoff).await;
|
||||
if let Some(deadline) = deadline {
|
||||
if time::timeout_at(deadline, time::sleep(backoff)).await.is_err() {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
} else {
|
||||
time::sleep(backoff).await;
|
||||
}
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
}
|
||||
other => return other,
|
||||
}
|
||||
@@ -957,32 +1012,35 @@ impl RemoteDisk {
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout occurred, mark disk as potentially faulty
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
"endpoint" => self.endpoint.to_string(),
|
||||
"op" => op.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
self.record_timeout(op, timeout_duration);
|
||||
if failure_health_action == FailureHealthAction::MarkFailure {
|
||||
self.mark_faulty_and_evict("operation_timeout").await;
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
state = "timeout",
|
||||
"Remote disk operation timed out"
|
||||
);
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
"endpoint" => self.endpoint.to_string(),
|
||||
"op" => op.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
state = "timeout",
|
||||
"Remote disk operation timed out"
|
||||
);
|
||||
}
|
||||
|
||||
async fn handle_network_like_error<T>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
@@ -1016,7 +1074,7 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_faulty_and_evict(&self, reason: &'static str) {
|
||||
fn mark_faulty(&self, reason: &'static str) -> bool {
|
||||
let previous_state = self.runtime_state();
|
||||
let transitioned_to_offline = self.mark_suspect_or_offline(reason);
|
||||
let state = self.runtime_state();
|
||||
@@ -1053,6 +1111,12 @@ impl RemoteDisk {
|
||||
"Remote disk marked suspect"
|
||||
);
|
||||
}
|
||||
}
|
||||
state != previous_state
|
||||
}
|
||||
|
||||
async fn mark_faulty_and_evict(&self, reason: &'static str) {
|
||||
if self.mark_faulty(reason) {
|
||||
counter!(
|
||||
"rustfs_drive_connection_evict_total",
|
||||
"endpoint" => self.endpoint.to_string(),
|
||||
@@ -1295,6 +1359,71 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
src_volume,
|
||||
src_path,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
op = "rename_data",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
|| async {
|
||||
let file_info = compat_json(fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
file_info,
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)?;
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -2068,7 +2197,7 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
Ok(file_info)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
get_drive_metadata_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2220,58 +2349,8 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
trace!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
src_volume,
|
||||
src_path,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
op = "rename_data",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
|| async {
|
||||
let file_info = compat_json(&fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(&fi)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
file_info,
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)?;
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -2417,6 +2496,30 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
let disk = self.disk_ref().await;
|
||||
let stall_timeout = get_object_disk_read_timeout();
|
||||
self.open_read_chunks_with_retry(ReadStreamRequest {
|
||||
endpoint: self.endpoint.grid_host(),
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
offset,
|
||||
length,
|
||||
stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout),
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Buffered read for remote disks.
|
||||
/// The transport stream is collected into owned Bytes for caller sharing.
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -5094,6 +5197,452 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_reset_during_backoff_preserves_recovery() {
|
||||
let remote_disk = Arc::new(new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await);
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let first_attempt = Arc::new(tokio::sync::Notify::new());
|
||||
let started = time::Instant::now();
|
||||
|
||||
let task_disk = Arc::clone(&remote_disk);
|
||||
let task_attempts = Arc::clone(&attempts);
|
||||
let task_first_attempt = Arc::clone(&first_attempt);
|
||||
let task = tokio::spawn(async move {
|
||||
task_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
move || {
|
||||
let attempt = task_attempts.fetch_add(1, Ordering::SeqCst);
|
||||
let first_attempt = Arc::clone(&task_first_attempt);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
time::sleep(Duration::from_millis(20)).await;
|
||||
first_attempt.notify_one();
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
first_attempt.notified().await;
|
||||
tokio::task::yield_now().await;
|
||||
remote_disk.health.reset_for_store_init_retry(&remote_disk.endpoint);
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
task.await
|
||||
.expect("retry task should finish")
|
||||
.expect("the retry should succeed after the health reset");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_millis(70));
|
||||
assert_eq!(
|
||||
remote_disk.health.waiting_count(),
|
||||
0,
|
||||
"health reset must not underflow the waiting counter"
|
||||
);
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(
|
||||
runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await,
|
||||
"a recovered channel must survive the retry backoff"
|
||||
);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_still_retries_within_shared_deadline() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
|
||||
remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
.expect("a retry that fits the shared deadline should succeed");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_uses_remaining_budget_for_final_attempt() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
time::sleep(Duration::from_millis(20)).await;
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)));
|
||||
}
|
||||
std::future::pending::<Result<()>>().await
|
||||
}
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
.expect_err("the final retry should consume only the remaining total budget");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_millis(100));
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_uses_final_attempt_at_exact_backoff_boundary() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
time::sleep(Duration::from_millis(50)).await;
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)));
|
||||
}
|
||||
std::future::pending::<Result<()>>().await
|
||||
}
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
.expect_err("the exact backoff boundary should be reserved for a final attempt");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_millis(100));
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_uses_final_attempt_below_backoff_budget() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
time::sleep(Duration::from_millis(80)).await;
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)));
|
||||
}
|
||||
std::future::pending::<Result<()>>().await
|
||||
}
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
.expect_err("remaining budget below backoff should be reserved for a final attempt");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_millis(100));
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_zero_timeout_disables_the_deadline() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
|
||||
remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.expect("zero timeout should allow a retry without a deadline");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), REMOTE_DISK_READ_RETRY_BASE_BACKOFF);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_accepts_max_metadata_timeout() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some(u64::MAX.to_string()))], async {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
|
||||
remote_disk
|
||||
.execute_read_with_retry("read_version", || async { Ok::<(), Error>(()) }, get_drive_metadata_timeout())
|
||||
.await
|
||||
.expect("the maximum configured metadata timeout must not panic");
|
||||
|
||||
remote_disk.cancel_token.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_zero_retries_runs_once() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_IDEMPOTENT_READ_RETRIES, Some("0"))], async {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async {
|
||||
Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
)))
|
||||
}
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect_err("zero retries should return the first network error");
|
||||
|
||||
assert!(matches!(err, DiskError::Io(ref io_err) if io_err.kind() == std_io::ErrorKind::ConnectionReset));
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(started.elapsed(), Duration::ZERO);
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
|
||||
remote_disk.cancel_token.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_attempt_timeout_marks_health_without_evicting() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let recorder = crate::test_metrics::CapturingRecorder::default();
|
||||
let _recorder_guard = metrics::set_default_local_recorder(&recorder);
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::pending::<Result<()>>()
|
||||
},
|
||||
Duration::from_millis(100),
|
||||
)
|
||||
.await
|
||||
.expect_err("an in-flight attempt that consumes the deadline should time out");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
assert!(runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
|
||||
assert_eq!(
|
||||
recorder.counter_value(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
&[
|
||||
("endpoint", remote_disk.endpoint.to_string().as_str()),
|
||||
("op", "read_version")
|
||||
]
|
||||
),
|
||||
1
|
||||
);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_does_not_retry_business_errors() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async { Err::<(), Error>(DiskError::FileNotFound) }
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect_err("business errors should be returned directly");
|
||||
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_honors_configured_retry_count() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_IDEMPOTENT_READ_RETRIES, Some("2"))], async {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let started = time::Instant::now();
|
||||
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
|
||||
.expect("remote disk address should parse")
|
||||
.connect_lazy();
|
||||
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
|
||||
|
||||
let err = remote_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
|| {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async {
|
||||
Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
)))
|
||||
}
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect_err("exhausted retries should return the last network error");
|
||||
|
||||
assert!(matches!(err, DiskError::Io(ref io_err) if io_err.kind() == std_io::ErrorKind::ConnectionReset));
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(started.elapsed(), Duration::from_millis(150));
|
||||
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
|
||||
remote_disk.cancel_token.cancel();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial(remote_disk_read_retry)]
|
||||
async fn execute_read_with_retry_stops_when_disk_turns_offline_during_backoff() {
|
||||
let remote_disk = Arc::new(new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await);
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let first_attempt = Arc::new(tokio::sync::Notify::new());
|
||||
let task_disk = Arc::clone(&remote_disk);
|
||||
let task_attempts = Arc::clone(&attempts);
|
||||
let task_first_attempt = Arc::clone(&first_attempt);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
task_disk
|
||||
.execute_read_with_retry(
|
||||
"read_version",
|
||||
move || {
|
||||
let attempt = task_attempts.fetch_add(1, Ordering::SeqCst);
|
||||
let first_attempt = Arc::clone(&task_first_attempt);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
first_attempt.notify_one();
|
||||
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
|
||||
std_io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
first_attempt.notified().await;
|
||||
tokio::task::yield_now().await;
|
||||
remote_disk
|
||||
.health
|
||||
.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
time::advance(REMOTE_DISK_READ_RETRY_BASE_BACKOFF).await;
|
||||
let err = task
|
||||
.await
|
||||
.expect("retry task should finish")
|
||||
.expect_err("an offline disk must stop before the next attempt");
|
||||
|
||||
assert_eq!(err, DiskError::FaultyDisk);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
remote_disk.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_evicts_cached_connection() {
|
||||
let addr = "http://127.0.0.1:59991".to_string();
|
||||
@@ -5603,6 +6152,40 @@ mod tests {
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else {
|
||||
return;
|
||||
};
|
||||
let remote_disk = remote_disk_for_addr(&base_addr).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("1")),
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("10")),
|
||||
],
|
||||
async {
|
||||
let started = time::Instant::now();
|
||||
let err = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
remote_disk.read_version("bucket", "bucket", "object", "", &ReadOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("read_version must use the shorter metadata deadline")
|
||||
.expect_err("a stalled peer must fail read_version");
|
||||
|
||||
assert!(matches!(err, DiskError::Timeout), "expected the metadata deadline to fire, got {err:?}");
|
||||
assert!(started.elapsed() >= Duration::from_millis(900));
|
||||
assert!(started.elapsed() < Duration::from_secs(2));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
remote_disk.cancel_token.cancel();
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_volume_bounds_the_wait_on_a_stalled_peer() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -48,10 +48,6 @@ impl RemoteClient {
|
||||
Self { addr: endpoint }
|
||||
}
|
||||
|
||||
pub fn from_url(url: url::Url) -> Self {
|
||||
Self { addr: url.to_string() }
|
||||
}
|
||||
|
||||
fn build_ping_request() -> PingRequest {
|
||||
let mut fbb = flatbuffers::FlatBufferBuilder::new();
|
||||
let payload = fbb.create_vector(b"health-check");
|
||||
|
||||
@@ -46,7 +46,6 @@ use rustfs_config::{
|
||||
SCANNER_SUB_SYS,
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::LazyLock;
|
||||
@@ -200,8 +199,6 @@ pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
|
||||
|
||||
pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES];
|
||||
|
||||
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
|
||||
|
||||
type ServerConfigDecryptFn = crate::bucket::migration::LegacyBlobDecryptFn;
|
||||
|
||||
static SERVER_CONFIG_DECRYPT_FN: LazyLock<RwLock<Option<ServerConfigDecryptFn>>> = LazyLock::new(|| RwLock::new(None));
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: configuration migration keeps legacy subsystem definitions available behind this module.
|
||||
#![allow(dead_code)]
|
||||
|
||||
mod audit;
|
||||
pub mod com;
|
||||
|
||||
@@ -101,6 +101,7 @@ const DEFAULT_RRS_STORAGE_CLASS: &str = "EC:1";
|
||||
const ZERO_SET_DRIVE_COUNT_ERROR: &str = "set drive count must be greater than zero";
|
||||
|
||||
pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
|
||||
const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
let kvs = vec![
|
||||
@@ -150,6 +151,8 @@ pub struct Config {
|
||||
optimize: Option<String>,
|
||||
inline_block: usize,
|
||||
initialized: bool,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
inline_block_explicit: bool,
|
||||
#[serde(skip)]
|
||||
standard_parities: Vec<PoolParity>,
|
||||
#[serde(skip)]
|
||||
@@ -186,6 +189,10 @@ impl Config {
|
||||
/// A topology-bound lookup fails closed for unknown drive counts and for
|
||||
/// deserialized legacy configurations that have no pool topology. Legacy
|
||||
/// callers retain scalar compatibility through [`Self::get_parity_for_sc`].
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "per-set parity resolution asserted by this file's tests (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn parity_for_sc(&self, sc: &str, drives_per_set: usize) -> Option<usize> {
|
||||
if !self.initialized {
|
||||
return None;
|
||||
@@ -233,17 +240,19 @@ impl Config {
|
||||
.map(|(pool_index, pool)| (pool_index, pool.drives_per_set))
|
||||
}
|
||||
|
||||
pub fn should_inline(&self, shard_size: i64, versioned: bool) -> bool {
|
||||
if shard_size < 0 {
|
||||
pub fn should_inline(&self, shard_size: i64, data_shards: usize, versioned: bool) -> bool {
|
||||
if shard_size < 0 || data_shards == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let shard_size = shard_size as usize;
|
||||
|
||||
let mut inline_block = DEFAULT_INLINE_BLOCK;
|
||||
if self.initialized {
|
||||
inline_block = self.inline_block;
|
||||
}
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
let inline_block = if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
};
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
@@ -392,6 +401,7 @@ fn lookup_config_for_pools_with_env(
|
||||
}
|
||||
|
||||
let optimize = overrides.optimize;
|
||||
let inline_block_explicit = overrides.inline_block.is_some();
|
||||
let inline_block = if let Some(value) = overrides.inline_block {
|
||||
let block = value
|
||||
.parse::<bytesize::ByteSize>()
|
||||
@@ -424,6 +434,7 @@ fn lookup_config_for_pools_with_env(
|
||||
optimize,
|
||||
inline_block,
|
||||
initialized: true,
|
||||
inline_block_explicit,
|
||||
standard_parities,
|
||||
rrs_parities,
|
||||
})
|
||||
@@ -541,22 +552,26 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_inline_preserves_exact_default_shard_boundaries() {
|
||||
let config = Config::default();
|
||||
fn should_inline_scales_default_threshold_by_data_shards() {
|
||||
let config = lookup_config_for_pools_with_env(&KVS::new(), &[3, 12], no_env_overrides())
|
||||
.expect("default inline policy should resolve for EC2+1 and EC8+4");
|
||||
|
||||
for (case, shard_size, versioned, expected) in [
|
||||
("unversioned below", 128 * 1024 - 1, false, true),
|
||||
("unversioned exact", 128 * 1024, false, true),
|
||||
("unversioned above", 128 * 1024 + 1, false, false),
|
||||
("versioned below", 16 * 1024 - 1, true, true),
|
||||
("versioned exact", 16 * 1024, true, true),
|
||||
("versioned above", 16 * 1024 + 1, true, false),
|
||||
("negative", -1, false, false),
|
||||
for (case, shard_size, data_shards, versioned, expected) in [
|
||||
("EC2+1 unversioned exact", 128 * 1024, 2, false, true),
|
||||
("EC2+1 unversioned above", 128 * 1024 + 1, 2, false, false),
|
||||
("EC2+1 versioned exact", 16 * 1024, 2, true, true),
|
||||
("EC2+1 versioned above", 16 * 1024 + 1, 2, true, false),
|
||||
("EC8+4 unversioned exact", 32 * 1024, 8, false, true),
|
||||
("EC8+4 unversioned above", 32 * 1024 + 1, 8, false, false),
|
||||
("EC8+4 versioned exact", 4 * 1024, 8, true, true),
|
||||
("EC8+4 versioned above", 4 * 1024 + 1, 8, true, false),
|
||||
("negative", -1, 2, false, false),
|
||||
("zero data shards", 0, 0, false, false),
|
||||
] {
|
||||
assert_eq!(
|
||||
config.should_inline(shard_size, versioned),
|
||||
config.should_inline(shard_size, data_shards, versioned),
|
||||
expected,
|
||||
"{case}: shard_size={shard_size}, versioned={versioned}"
|
||||
"{case}: shard_size={shard_size}, data_shards={data_shards}, versioned={versioned}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -577,13 +592,28 @@ mod tests {
|
||||
let shard_size = erasure.shard_file_size(object_size);
|
||||
assert_eq!(shard_size, expected_shard_size, "{case}: object_size={object_size}");
|
||||
assert_eq!(
|
||||
config.should_inline(shard_size, versioned),
|
||||
config.should_inline(shard_size, erasure.data_shards, versioned),
|
||||
expected,
|
||||
"{case}: object_size={object_size}, shard_size={shard_size}, versioned={versioned}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_inline_block_preserves_fixed_per_shard_rollback() {
|
||||
let overrides = StorageClassEnvOverrides {
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let config = lookup_config_for_pools_with_env(&KVS::new(), &[12], overrides)
|
||||
.expect("explicit inline block should resolve for EC8+4");
|
||||
|
||||
assert!(config.should_inline(128 * 1024, 8, false));
|
||||
assert!(!config.should_inline(128 * 1024 + 1, 8, false));
|
||||
assert!(config.should_inline(16 * 1024, 8, true));
|
||||
assert!(!config.should_inline(16 * 1024 + 1, 8, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_capability_contract_only_accepts_implemented_layouts() {
|
||||
assert_eq!(SUPPORTED_WRITE_CLASSES, [STANDARD, RRS]);
|
||||
@@ -777,6 +807,7 @@ mod tests {
|
||||
let encoded = serde_json::to_string(&cfg).expect("config should serialize");
|
||||
assert!(!encoded.contains("standard_parities"));
|
||||
assert!(!encoded.contains("rrs_parities"));
|
||||
assert!(!encoded.contains("inline_block_explicit"));
|
||||
|
||||
let decoded: Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize");
|
||||
assert_eq!(decoded.get_parity_for_sc(STANDARD), Some(2));
|
||||
@@ -786,6 +817,25 @@ mod tests {
|
||||
assert!(validate_parity(0, 0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_inline_block_survives_config_round_trip() {
|
||||
let cfg = lookup_config_for_pools_with_env(
|
||||
&KVS::new(),
|
||||
&[12],
|
||||
StorageClassEnvOverrides {
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("explicit inline block should resolve");
|
||||
assert!(cfg.should_inline(100 * 1024, 8, false));
|
||||
|
||||
let encoded = serde_json::to_string(&cfg).expect("config should serialize");
|
||||
assert!(encoded.contains("\"inline_block_explicit\":true"));
|
||||
let decoded: Config = serde_json::from_str(&encoded).expect("explicit inline config should deserialize");
|
||||
assert!(decoded.should_inline(100 * 1024, 8, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_config_reads_rrs_from_class_rrs_key() {
|
||||
// Regression: kvs.get(RRS) used RRS="REDUCED_REDUNDANCY" instead of
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: pool coordination helpers are being migrated behind runtime owners.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub(crate) mod pools;
|
||||
pub(crate) mod sets;
|
||||
|
||||
@@ -226,6 +226,7 @@ fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>
|
||||
ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")]
|
||||
fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
|
||||
for idx in indices {
|
||||
ensure_local_decommission_pool_leader(endpoints, *idx)?;
|
||||
@@ -1058,11 +1059,19 @@ fn should_cleanup_decommission_source_entry(decommissioned: usize, total_version
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
|
||||
)]
|
||||
enum DecommissionTerminalState {
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
|
||||
)]
|
||||
fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState {
|
||||
if failed_items_present {
|
||||
DecommissionTerminalState::Failed
|
||||
@@ -2266,15 +2275,19 @@ fn decommission_delete_marker_opts(
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
version_id: Option<String>,
|
||||
src_pool_idx: usize,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> ObjectOptions {
|
||||
let version_suspended = version.version_id.is_none() && version_id.is_none();
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id,
|
||||
versioned: !version_suspended,
|
||||
version_suspended,
|
||||
version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
|
||||
mod_time: version.mod_time,
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
delete_marker: true,
|
||||
skip_decommissioned: true,
|
||||
expected_bucket_incarnation_id,
|
||||
delete_replication: version
|
||||
.replication_state_internal
|
||||
.as_ref()
|
||||
@@ -2299,6 +2312,7 @@ fn decommission_remote_tiered_opts(
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
version_id: Option<String>,
|
||||
src_pool_idx: usize,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: version_id.is_some(),
|
||||
@@ -2307,6 +2321,9 @@ fn decommission_remote_tiered_opts(
|
||||
user_defined: version.metadata.clone(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
include_part_checksums: true,
|
||||
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -2805,6 +2822,7 @@ impl ECStore {
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
@@ -2834,6 +2852,11 @@ impl ECStore {
|
||||
}
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
|
||||
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
|
||||
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
|
||||
|
||||
fivs.versions
|
||||
@@ -2894,7 +2917,7 @@ impl ECStore {
|
||||
.delete_object(
|
||||
bucket.as_str(),
|
||||
&version.name,
|
||||
decommission_delete_marker_opts(version, version_id.clone(), idx),
|
||||
decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2984,7 +3007,7 @@ impl ECStore {
|
||||
bucket.as_str(),
|
||||
&version.name,
|
||||
version,
|
||||
&decommission_remote_tiered_opts(version, version_id.clone(), idx),
|
||||
&decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -3056,7 +3079,11 @@ impl ECStore {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await {
|
||||
if let Err(err) = self
|
||||
.clone()
|
||||
.decommission_object(idx, bucket, rd, expected_bucket_incarnation_id)
|
||||
.await
|
||||
{
|
||||
if is_decommission_copy_cleanup_safe_error(&err) {
|
||||
ignore = true;
|
||||
cleanup_ignored = true;
|
||||
@@ -3133,6 +3160,9 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
|
||||
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
|
||||
}
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
|
||||
self.save_decommission_entry_progress_stage(
|
||||
@@ -3157,6 +3187,12 @@ impl ECStore {
|
||||
entry.name.as_str(),
|
||||
&fivs,
|
||||
&cleanup_preflight_allowed_missing,
|
||||
data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: expected_bucket_incarnation_id,
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
},
|
||||
"decommission",
|
||||
)
|
||||
.await
|
||||
@@ -3268,6 +3304,11 @@ impl ECStore {
|
||||
let mut lifecycle_config = None;
|
||||
let mut object_lock_config = None;
|
||||
let mut replication_config = None;
|
||||
let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
|
||||
None
|
||||
} else {
|
||||
Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
|
||||
};
|
||||
|
||||
if bi.name != RUSTFS_META_BUCKET {
|
||||
let _ = resolve_decommission_optional_bucket_config_result(
|
||||
@@ -3321,6 +3362,7 @@ impl ECStore {
|
||||
let lifecycle_config = lifecycle_config.clone();
|
||||
let object_lock_config = object_lock_config.clone();
|
||||
let replication_config = replication_config.clone();
|
||||
let expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
||||
let entry_error = entry_error.clone();
|
||||
let callback_rx = callback_rx.clone();
|
||||
|
||||
@@ -3383,6 +3425,7 @@ impl ECStore {
|
||||
lifecycle_config,
|
||||
object_lock_config,
|
||||
replication_config,
|
||||
expected_bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -4168,10 +4211,24 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, rd))]
|
||||
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
|
||||
async fn decommission_object(
|
||||
self: Arc<Self>,
|
||||
pool_idx: usize,
|
||||
bucket: String,
|
||||
rd: GetObjectReader,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> Result<()> {
|
||||
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
|
||||
let object_name = rd.object_info.name.clone();
|
||||
let result = data_movement::migrate_object(self, pool_idx, bucket.clone(), rd, "decommission_object").await;
|
||||
let result = data_movement::migrate_object(
|
||||
self,
|
||||
pool_idx,
|
||||
bucket.clone(),
|
||||
rd,
|
||||
expected_bucket_incarnation_id,
|
||||
"decommission_object",
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
warn!("decommission_object: migrated {} {}", &bucket, &object_name);
|
||||
}
|
||||
@@ -4347,7 +4404,8 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7);
|
||||
let incarnation = uuid::Uuid::new_v4();
|
||||
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
|
||||
let replication = opts.delete_replication.expect("replication state should be preserved");
|
||||
|
||||
assert!(opts.versioned);
|
||||
@@ -4357,11 +4415,25 @@ mod tests {
|
||||
assert_eq!(opts.src_pool_idx, 7);
|
||||
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
||||
assert_eq!(opts.mod_time, Some(mod_time));
|
||||
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
|
||||
assert!(replication.delete_marker);
|
||||
assert_eq!(replication.replicate_decision_str, "existing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_delete_marker_opts_preserves_suspended_null_version() {
|
||||
let version = rustfs_filemeta::FileInfo {
|
||||
deleted: true,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = decommission_delete_marker_opts(&version, None, 7, None);
|
||||
|
||||
assert!(!opts.versioned);
|
||||
assert!(opts.version_suspended);
|
||||
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
|
||||
let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
|
||||
@@ -4383,7 +4455,8 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9);
|
||||
let incarnation = uuid::Uuid::new_v4();
|
||||
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
|
||||
|
||||
assert!(opts.versioned);
|
||||
assert!(opts.data_movement);
|
||||
@@ -4391,6 +4464,9 @@ mod tests {
|
||||
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
||||
assert_eq!(opts.mod_time, Some(mod_time));
|
||||
assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
|
||||
assert!(opts.include_part_checksums);
|
||||
assert!(opts.http_preconditions.is_some());
|
||||
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+1564
-253
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-disk usage snapshots persisted under the metadata bucket.
|
||||
//!
|
||||
//! **Nothing calls into this module.** It landed complete with tests in #5307
|
||||
//! (2026-07-27) and its aggregation entry point,
|
||||
//! [`crate::data_usage::aggregate_local_snapshots`], has never had a caller in
|
||||
//! the tree's history. The live data-usage path is
|
||||
//! `load_data_usage_from_backend` / `store_data_usage_in_backend`. The items
|
||||
//! below therefore carry individual `dead_code` allows rather than a module
|
||||
//! blanket, so the gap stays greppable until it is either wired up or removed.
|
||||
|
||||
use crate::data_usage::BucketUsageInfo;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -26,10 +36,12 @@ pub const DATA_USAGE_DIR: &str = "datausage";
|
||||
/// Directory used to store incremental scan state files under the metadata bucket.
|
||||
pub const DATA_USAGE_STATE_DIR: &str = "datausage/state";
|
||||
/// Snapshot file format version, allows forward compatibility if the structure evolves.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub const LOCAL_USAGE_SNAPSHOT_VERSION: u32 = 1;
|
||||
|
||||
/// Additional metadata describing which disk produced the snapshot.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub struct LocalUsageSnapshotMeta {
|
||||
/// Disk UUID stored as a string for simpler serialization.
|
||||
pub disk_id: String,
|
||||
@@ -43,6 +55,7 @@ pub struct LocalUsageSnapshotMeta {
|
||||
|
||||
/// Usage snapshot produced by a single disk.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub struct LocalUsageSnapshot {
|
||||
/// Format version recorded in the snapshot.
|
||||
pub format_version: u32,
|
||||
@@ -64,6 +77,7 @@ pub struct LocalUsageSnapshot {
|
||||
pub objects_total_size: u64,
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
impl LocalUsageSnapshot {
|
||||
/// Create an empty snapshot with the default format version filled in.
|
||||
pub fn new(meta: LocalUsageSnapshotMeta) -> Self {
|
||||
@@ -99,11 +113,13 @@ impl LocalUsageSnapshot {
|
||||
}
|
||||
|
||||
/// Build the snapshot file name `<disk-id>.json`.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub fn snapshot_file_name(disk_id: &str) -> String {
|
||||
format!("{disk_id}.json")
|
||||
}
|
||||
|
||||
/// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/<disk-id>.json`.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub fn snapshot_object_path(disk_id: &str) -> String {
|
||||
format!("{}/{}", DATA_USAGE_DIR, snapshot_file_name(disk_id))
|
||||
}
|
||||
@@ -119,11 +135,13 @@ pub fn data_usage_state_dir(root: &Path) -> PathBuf {
|
||||
}
|
||||
|
||||
/// Build the absolute path to the snapshot file for the provided disk ID.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub fn snapshot_path(root: &Path, disk_id: &str) -> PathBuf {
|
||||
data_usage_dir(root).join(snapshot_file_name(disk_id))
|
||||
}
|
||||
|
||||
/// Read a snapshot from disk if it exists.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result<Option<LocalUsageSnapshot>> {
|
||||
let path = snapshot_path(root, disk_id);
|
||||
match fs::read(&path).await {
|
||||
@@ -138,6 +156,7 @@ pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result<Option<LocalUsa
|
||||
}
|
||||
|
||||
/// Persist a snapshot to disk, creating directories as needed and overwriting any existing file.
|
||||
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
|
||||
pub async fn write_snapshot(root: &Path, disk_id: &str, snapshot: &LocalUsageSnapshot) -> Result<()> {
|
||||
let dir = data_usage_dir(root);
|
||||
fs::create_dir_all(&dir).await.map_err(Error::other)?;
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: scanner/data-usage state is partially migrated and still owns staged cache helpers.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod local_snapshot;
|
||||
|
||||
@@ -34,8 +33,8 @@ use crate::{
|
||||
pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
|
||||
use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
|
||||
VersionsHistogram, observed_data_usage_is_newer,
|
||||
DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
|
||||
observed_data_usage_is_newer,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
@@ -55,7 +54,6 @@ use tracing::{debug, error, info, instrument};
|
||||
// Data usage storage constants
|
||||
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
|
||||
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
|
||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
|
||||
const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024;
|
||||
@@ -313,11 +311,6 @@ lazy_static::lazy_static! {
|
||||
LEGACY_DATA_USAGE_OBJECT_NAME
|
||||
);
|
||||
static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
SLASH_SEPARATOR,
|
||||
DATA_USAGE_BLOOM_NAME
|
||||
);
|
||||
pub static ref DATA_COMPRESSION_TOTAL_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
SLASH_SEPARATOR,
|
||||
@@ -858,6 +851,10 @@ async fn resolve_loaded_snapshot_pair_with_source(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "primary/backup snapshot fallback asserted by this file's tests (backlog#1823)"
|
||||
)]
|
||||
async fn resolve_loaded_snapshot(
|
||||
primary: Result<Vec<u8>, Error>,
|
||||
backup: impl Future<Output = Result<Vec<u8>, Error>>,
|
||||
@@ -1187,6 +1184,10 @@ pub async fn invalidate_admin_data_usage_snapshot_cache() {
|
||||
}
|
||||
|
||||
/// Aggregate usage information from local disk snapshots.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only through aggregate_local_snapshots, which has no caller (backlog#1823)"
|
||||
)]
|
||||
fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
|
||||
if let Some(update) = snapshot.last_update
|
||||
&& latest_update.is_none_or(|current| update > current)
|
||||
@@ -1220,6 +1221,10 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "entry point of the local usage-snapshot feature, which has had no caller since it landed in #5307 (backlog#1823)"
|
||||
)]
|
||||
pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskUsageStatus>, DataUsageInfo), Error> {
|
||||
let mut aggregated = DataUsageInfo::default();
|
||||
let mut latest_update: Option<SystemTime> = None;
|
||||
@@ -1355,7 +1360,7 @@ impl BucketUsageAccumulator {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let object_size = object.size.max(0) as u64;
|
||||
let object_size = quota_object_size(object)?;
|
||||
self.current_live_versions = self.current_live_versions.saturating_add(1);
|
||||
self.size_histogram.add(object_size);
|
||||
self.total_size = self.total_size.saturating_add(object_size);
|
||||
@@ -1385,6 +1390,31 @@ impl BucketUsageAccumulator {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let persisted_part_size = if object.parts.is_empty() {
|
||||
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
} else {
|
||||
object.parts.iter().try_fold(0_u64, |total, part| {
|
||||
// Compressed streaming objects persist -1 when the transformed
|
||||
// part size is unknown. The physical part size remains a valid
|
||||
// quota floor; reject only non-negative values that overflow.
|
||||
let actual_size = if part.actual_size < 0 {
|
||||
if object.is_compressed() {
|
||||
0
|
||||
} else {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
} else {
|
||||
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
};
|
||||
let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?);
|
||||
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
|
||||
})?
|
||||
};
|
||||
Ok(logical_size.max(persisted_part_size))
|
||||
}
|
||||
|
||||
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
|
||||
pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
|
||||
@@ -1742,11 +1772,6 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne
|
||||
entry.pending_scanner_position = None;
|
||||
}
|
||||
|
||||
/// Fast in-memory increment for immediate quota consistency.
|
||||
pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) {
|
||||
record_bucket_object_write_memory(bucket, None, size_increment).await;
|
||||
}
|
||||
|
||||
/// Fast in-memory update for successful object deletes.
|
||||
pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
|
||||
ensure_bucket_usage_cached(bucket).await;
|
||||
@@ -1789,11 +1814,6 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) {
|
||||
entry.pending_scanner_position = None;
|
||||
}
|
||||
|
||||
/// Fast in-memory decrement for immediate quota consistency
|
||||
pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
|
||||
record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await;
|
||||
}
|
||||
|
||||
/// Get bucket usage from the authoritative cache for this topology.
|
||||
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
|
||||
let store = runtime_sources::object_store_handle()?;
|
||||
@@ -1988,91 +2008,6 @@ pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageIn
|
||||
apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await;
|
||||
}
|
||||
|
||||
/// Sync memory cache with backend data (called by scanner)
|
||||
pub async fn sync_memory_cache_with_backend() -> Result<(), Error> {
|
||||
if let Some(store) = runtime_sources::object_store_handle() {
|
||||
match load_data_usage_from_backend(store.clone()).await {
|
||||
Ok(data_usage_info) => {
|
||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to sync memory cache with backend: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a data usage cache entry from size summary
|
||||
pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
entry.add_sizes(summary);
|
||||
entry
|
||||
}
|
||||
|
||||
/// Convert data usage cache to DataUsageInfo
|
||||
pub fn cache_to_data_usage_info(
|
||||
cache: &DataUsageCache,
|
||||
path: &str,
|
||||
buckets: &[crate::storage_api_contracts::bucket::BucketInfo],
|
||||
) -> DataUsageInfo {
|
||||
let e = match cache.find(path) {
|
||||
Some(e) => e,
|
||||
None => return DataUsageInfo::default(),
|
||||
};
|
||||
let flat = cache.flatten(&e);
|
||||
|
||||
let mut buckets_usage = HashMap::new();
|
||||
for bucket in buckets.iter() {
|
||||
let e = match cache.find(&bucket.name) {
|
||||
Some(e) => e,
|
||||
None => continue,
|
||||
};
|
||||
let flat = cache.flatten(&e);
|
||||
let mut bui = BucketUsageInfo {
|
||||
size: flat.size as u64,
|
||||
versions_count: flat.versions as u64,
|
||||
objects_count: flat.objects as u64,
|
||||
delete_markers_count: flat.delete_markers as u64,
|
||||
object_size_histogram: flat.obj_sizes.to_map(),
|
||||
object_versions_histogram: flat.obj_versions.to_map(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(rs) = &flat.replication_stats {
|
||||
bui.replica_size = rs.replica_size;
|
||||
bui.replica_count = rs.replica_count;
|
||||
|
||||
for (arn, stat) in rs.targets.iter() {
|
||||
bui.replication_info.insert(
|
||||
arn.clone(),
|
||||
BucketTargetUsageInfo {
|
||||
replication_pending_size: stat.pending_size,
|
||||
replicated_size: stat.replicated_size,
|
||||
replication_failed_size: stat.failed_size,
|
||||
replication_pending_count: stat.pending_count,
|
||||
replication_failed_count: stat.failed_count,
|
||||
replicated_count: stat.replicated_count,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
buckets_usage.insert(bucket.name.clone(), bui);
|
||||
}
|
||||
|
||||
DataUsageInfo {
|
||||
last_update: cache.info.last_update,
|
||||
objects_total_count: flat.objects as u64,
|
||||
versions_total_count: flat.versions as u64,
|
||||
delete_markers_total_count: flat.delete_markers as u64,
|
||||
objects_total_size: flat.size as u64,
|
||||
buckets_count: e.children.len() as u64,
|
||||
buckets_usage,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions for DataUsageCache operations
|
||||
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
@@ -3124,6 +3059,102 @@ mod tests {
|
||||
assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_usage_uses_the_larger_of_logical_and_physical_size() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string());
|
||||
let object = ObjectInfo {
|
||||
name: "compressed".to_string(),
|
||||
size: 128,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
let mut usage = BucketUsageAccumulator::default();
|
||||
usage
|
||||
.record("bucket", &object)
|
||||
.expect("valid compressed metadata should be counted");
|
||||
assert_eq!(usage.finish().size, 4096);
|
||||
|
||||
let mut framed_metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut framed_metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||
let framed = ObjectInfo {
|
||||
name: "framed".to_string(),
|
||||
size: 17,
|
||||
user_defined: Arc::new(framed_metadata),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17);
|
||||
|
||||
let legacy_compressed_part = ObjectInfo {
|
||||
name: "legacy-compressed-part".to_string(),
|
||||
size: 1,
|
||||
user_defined: Arc::new((*framed.user_defined).clone()),
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 1,
|
||||
actual_size: -1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
quota_object_size(&legacy_compressed_part).expect("unknown compressed part size is a valid sentinel"),
|
||||
1
|
||||
);
|
||||
|
||||
let uncompressed_negative_part = ObjectInfo {
|
||||
name: "uncompressed-negative-part".to_string(),
|
||||
size: 1,
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 1,
|
||||
actual_size: -1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(quota_object_size(&uncompressed_negative_part), Err(Error::PartMissingOrCorrupt)));
|
||||
|
||||
let mut corrupt_metadata = (*object.user_defined).clone();
|
||||
rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string());
|
||||
let corrupt = ObjectInfo {
|
||||
user_defined: Arc::new(corrupt_metadata),
|
||||
..object
|
||||
};
|
||||
assert!(matches!(quota_object_size(&corrupt), Err(Error::PartMissingOrCorrupt)));
|
||||
|
||||
let mut poisoned_metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut poisoned_metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||
let poisoned = ObjectInfo {
|
||||
name: "legacy-swift-metadata".to_string(),
|
||||
size: 4096,
|
||||
user_defined: Arc::new(poisoned_metadata),
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 4096,
|
||||
actual_size: 4096,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"),
|
||||
4096
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
|
||||
|
||||
@@ -653,6 +653,7 @@ fn reconcile_servers_with_endpoint_topology(
|
||||
(added, report)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "exercised by this file's topology tests (backlog#1823)")]
|
||||
fn server_topology_completeness_report(
|
||||
servers: &[ServerProperties],
|
||||
endpoints: &EndpointServerPools,
|
||||
|
||||
@@ -46,21 +46,49 @@ pub(crate) const GET_CODEC_STREAMING_OBJECT_CLASS_MULTIPART: &str = "multipart";
|
||||
pub(crate) const GET_STAGE_DECODE: &str = "decode";
|
||||
pub(crate) const GET_STAGE_EMIT: &str = "emit";
|
||||
pub(crate) const GET_STAGE_FILL: &str = "fill";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_FIRST_BYTE: &str = "first_byte";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_FIRST_METADATA_RESPONSE: &str = "first_metadata_response";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_FIRST_VALID_METADATA_RESPONSE: &str = "first_valid_metadata_response";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_FIRST_SHARD_READ: &str = "first_shard_read";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_FULL_BODY: &str = "full_body";
|
||||
pub(crate) const GET_STAGE_INLINE_PREPARE: &str = "inline_prepare";
|
||||
pub(crate) const GET_STAGE_LOCK_ACQUIRE: &str = "lock_acquire";
|
||||
pub(crate) const GET_STAGE_METADATA: &str = "metadata";
|
||||
pub(crate) const GET_STAGE_METADATA_CACHE_LOOKUP: &str = "metadata_cache_lookup";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_METADATA_FANOUT: &str = "metadata_fanout";
|
||||
pub(crate) const GET_STAGE_METADATA_RESOLVE: &str = "metadata_resolve";
|
||||
pub(crate) const GET_STAGE_OBJECT_INFO: &str = "object_info";
|
||||
pub(crate) const GET_STAGE_OUTPUT_LOCK_WAIT: &str = "output_lock_wait";
|
||||
pub(crate) const GET_STAGE_OUTPUT_POLL: &str = "output_poll";
|
||||
pub(crate) const GET_STAGE_PATH_DECISION: &str = "path_decision";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_QUORUM_REACHED: &str = "quorum_reached";
|
||||
pub(crate) const GET_STAGE_RANGE: &str = "range";
|
||||
pub(crate) const GET_STAGE_READER_SETUP: &str = "reader_setup";
|
||||
@@ -84,12 +112,28 @@ pub(crate) const GET_STAGE_READER_STREAM_FIRST_READ: &str = "reader_stream_first
|
||||
pub(crate) const GET_STAGE_READER_TASK_BITROT_READER_INIT: &str = "reader_task_bitrot_reader_init";
|
||||
pub(crate) const GET_STAGE_READER_TASK_FILE_OPEN: &str = "reader_task_file_open";
|
||||
pub(crate) const GET_STAGE_READER_TASK_READER_CONSTRUCTION: &str = "reader_task_reader_construction";
|
||||
pub(crate) const GET_STAGE_READ_VERSION_DECODE: &str = "read_version_decode";
|
||||
pub(crate) const GET_STAGE_READ_VERSION_PATH_CHECK: &str = "read_version_path_check";
|
||||
pub(crate) const GET_STAGE_READ_VERSION_PATH_RESOLVE: &str = "read_version_path_resolve";
|
||||
pub(crate) const GET_STAGE_READ_VERSION_XLMETA_READ: &str = "read_version_xlmeta_read";
|
||||
pub(crate) const GET_STAGE_RECONSTRUCT: &str = "reconstruct";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_RESPONSE_HANDOFF: &str = "response_handoff";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_SLOWEST_METADATA_RESPONSE: &str = "slowest_metadata_response";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ: &str = "stripe_read";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ_FIRST_SHARD: &str = "stripe_read_first_shard";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ_QUORUM: &str = "stripe_read_quorum";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const GET_STAGE_BITROT_VERIFY: &str = "bitrot_verify";
|
||||
|
||||
pub(crate) const GET_READER_BUFFER_OUTPUT: &str = "output";
|
||||
@@ -137,6 +181,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_PART_CHECKSUMS: &str = "part_checksums";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
|
||||
@@ -145,6 +190,17 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
||||
@@ -154,8 +210,20 @@ pub(crate) const GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND: &str = "versi
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM: &str = "version_match_quorum";
|
||||
|
||||
/// Early-stop active state labels
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const EARLY_STOP_ACTIVE_HIT: &str = "hit";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const EARLY_STOP_ACTIVE_MISS: &str = "miss";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const EARLY_STOP_ACTIVE_DISABLED: &str = "disabled";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -441,6 +509,10 @@ mod tests {
|
||||
assert_eq!(GET_STAGE_QUORUM_REACHED, "quorum_reached");
|
||||
assert_eq!(GET_STAGE_RANGE, "range");
|
||||
assert_eq!(GET_STAGE_READER_SETUP, "reader_setup");
|
||||
assert_eq!(GET_STAGE_READ_VERSION_DECODE, "read_version_decode");
|
||||
assert_eq!(GET_STAGE_READ_VERSION_PATH_CHECK, "read_version_path_check");
|
||||
assert_eq!(GET_STAGE_READ_VERSION_PATH_RESOLVE, "read_version_path_resolve");
|
||||
assert_eq!(GET_STAGE_READ_VERSION_XLMETA_READ, "read_version_xlmeta_read");
|
||||
assert_eq!(GET_STAGE_RECONSTRUCT, "reconstruct");
|
||||
assert_eq!(GET_STAGE_RESPONSE_HANDOFF, "response_handoff");
|
||||
assert_eq!(GET_STAGE_SLOWEST_METADATA_RESPONSE, "slowest_metadata_response");
|
||||
@@ -480,6 +552,7 @@ mod tests {
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_PART_CHECKSUMS, "part_checksums");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication");
|
||||
@@ -489,6 +562,32 @@ mod tests {
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
||||
assert_eq!(
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||
"data_read_inline_body_verify"
|
||||
);
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
|
||||
assert_eq!(
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||
"data_read_inline_identity_mismatch"
|
||||
);
|
||||
assert_eq!(
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||
"data_read_inline_missing_payload"
|
||||
);
|
||||
assert_eq!(
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
|
||||
"data_read_inline_missing_shard"
|
||||
);
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
|
||||
assert_eq!(
|
||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||
"data_read_inline_transformed"
|
||||
);
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
||||
|
||||
@@ -13,8 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: diagnostics constants are staged for request-path telemetry migration.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub(crate) mod admin_server_info;
|
||||
pub(crate) mod get;
|
||||
pub(crate) mod pool;
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! BytesPool metric label constants.
|
||||
//!
|
||||
//! These constants are used when recording pool acquisition and return
|
||||
//! metrics to avoid string allocations and ensure label consistency.
|
||||
|
||||
/// BytesPool tier labels
|
||||
pub const POOL_TIER_SMALL: &str = "small";
|
||||
pub const POOL_TIER_MEDIUM: &str = "medium";
|
||||
pub const POOL_TIER_LARGE: &str = "large";
|
||||
pub const POOL_TIER_XLARGE: &str = "xlarge";
|
||||
|
||||
/// BytesPool outcome labels
|
||||
pub const POOL_OUTCOME_HIT: &str = "hit";
|
||||
pub const POOL_OUTCOME_MISS: &str = "miss";
|
||||
pub const POOL_OUTCOME_RECYCLED: &str = "recycled";
|
||||
pub const POOL_OUTCOME_DROPPED: &str = "dropped";
|
||||
@@ -241,6 +241,40 @@ pub fn get_drive_list_dir_timeout() -> Duration {
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) trait DiskStoreRenameDataExt {
|
||||
async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp>;
|
||||
}
|
||||
|
||||
impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
self.disk
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
@@ -603,14 +637,23 @@ impl Default for DiskOperationMetrics {
|
||||
}
|
||||
|
||||
impl DiskOperationMetrics {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||
)]
|
||||
fn record_call(&mut self) {
|
||||
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||
)]
|
||||
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
||||
self.record_latency_atomic(now_sec, elapsed);
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
|
||||
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
||||
self.record_call();
|
||||
self.record_latency(now_sec, elapsed);
|
||||
@@ -736,6 +779,7 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
/// Set disk as faulty
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn set_faulty(&self) {
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
}
|
||||
@@ -816,6 +860,7 @@ impl DiskHealthTracker {
|
||||
became_offline
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let current = self.runtime_state();
|
||||
if current == RuntimeDriveHealthState::Offline {
|
||||
@@ -946,11 +991,13 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
/// Get waiting operations count
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn waiting_count(&self) -> u32 {
|
||||
self.waiting.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Get last success timestamp
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn last_success(&self) -> i64 {
|
||||
self.last_success.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -992,21 +1039,6 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
/// Health check context key for tracking disk operations
|
||||
#[derive(Debug, Clone)]
|
||||
struct HealthDiskCtxKey;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HealthDiskCtxValue {
|
||||
last_success: Arc<AtomicI64>,
|
||||
}
|
||||
|
||||
impl HealthDiskCtxValue {
|
||||
fn log_success(&self) {
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1038,10 +1070,6 @@ impl LocalDiskWrapper {
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
|
||||
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_reconnect_state(
|
||||
disk: Arc<LocalDisk>,
|
||||
health_check: bool,
|
||||
@@ -1404,20 +1432,6 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
|
||||
if want_id.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let stored_disk_id = self.disk.get_disk_id().await?;
|
||||
|
||||
if stored_disk_id != want_id {
|
||||
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if disk ID is stale
|
||||
async fn check_disk_stale(&self) -> Result<()> {
|
||||
let Some(current_disk_id) = *self.disk_id.read().await else {
|
||||
@@ -1985,13 +1999,8 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||
@@ -2022,6 +2031,21 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_mmap_copy",
|
||||
|
||||
@@ -48,6 +48,7 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
||||
match io_err.kind() {
|
||||
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
||||
|
||||
@@ -178,6 +178,7 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
|
||||
@@ -15,6 +15,11 @@
|
||||
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
|
||||
use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
|
||||
use crate::diagnostics::get::{
|
||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READ_VERSION_DECODE,
|
||||
GET_STAGE_READ_VERSION_PATH_CHECK, GET_STAGE_READ_VERSION_PATH_RESOLVE, GET_STAGE_READ_VERSION_XLMETA_READ,
|
||||
get_stage_timer_if_enabled, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::disk::HEALING_MARKER_PATH;
|
||||
use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout};
|
||||
@@ -22,17 +27,18 @@ use crate::disk::{
|
||||
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo,
|
||||
DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize,
|
||||
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
conv_part_err_to_int,
|
||||
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
|
||||
endpoint::Endpoint,
|
||||
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
|
||||
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
|
||||
format::FormatV3,
|
||||
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
|
||||
os,
|
||||
is_quota_mutation_fence_path, os,
|
||||
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
|
||||
quota_mutation_fence_path,
|
||||
};
|
||||
use crate::erasure::coding::{self, bitrot_verify};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -55,9 +61,7 @@ use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Debug;
|
||||
use std::io::{Error as IoError, SeekFrom};
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
@@ -661,6 +665,7 @@ async fn remove_empty_directory_tree_under_mount_lease(
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn remove_empty_directory_tree_with(
|
||||
root: &Path,
|
||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
@@ -1012,13 +1017,29 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
|
||||
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
||||
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
||||
/// Default: false (uses page cache via mmap/pread).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
||||
|
||||
/// Minimum shard size threshold for O_DIRECT reads.
|
||||
/// Only shards larger than this threshold will use O_DIRECT.
|
||||
/// Default: 4MB.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
||||
@@ -1032,7 +1053,15 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
||||
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
||||
/// Default: false (buffered writes via the page cache, as before).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||
@@ -1091,12 +1120,14 @@ macro_rules! cached_read_env {
|
||||
|
||||
cached_read_env! {
|
||||
/// Check if O_DIRECT reads are enabled.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn is_direct_io_read_enabled() -> bool =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
||||
}
|
||||
|
||||
cached_read_env! {
|
||||
/// Check if O_DIRECT shard/part data writes are enabled.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn is_direct_io_write_enabled() -> bool =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
||||
}
|
||||
@@ -1452,6 +1483,7 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
||||
|
||||
cached_read_env! {
|
||||
/// Get the O_DIRECT read threshold size.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn get_direct_io_read_threshold() -> usize =
|
||||
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
||||
}
|
||||
@@ -1669,12 +1701,20 @@ impl DirectIoWriteState {
|
||||
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
||||
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
||||
/// into one positioned write to keep the syscall count low.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
||||
|
||||
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
||||
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
||||
/// flushed batch length, and every write offset stay alignment-correct.
|
||||
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
fn direct_write_staging_capacity(align: usize) -> usize {
|
||||
debug_assert!(align.is_power_of_two() && align >= 512);
|
||||
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
||||
@@ -1683,6 +1723,10 @@ fn direct_write_staging_capacity(align: usize) -> usize {
|
||||
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
||||
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
||||
/// so the tail-boundary math is unit-tested on any host.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
||||
let aligned = filled - (filled % align);
|
||||
(aligned, filled - aligned)
|
||||
@@ -2024,14 +2068,17 @@ static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex<Option<(Strin
|
||||
#[cfg(test)]
|
||||
type InlinePreparationHook = Box<dyn FnOnce() + Send>;
|
||||
#[cfg(test)]
|
||||
type RenameDataPublicationHookKey = (PathBuf, String, String);
|
||||
#[cfg(test)]
|
||||
static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock<
|
||||
std::sync::Mutex<HashMap<RenameDataPublicationHookKey, InlinePreparationHook>>,
|
||||
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static OWNED_FILE_WRITE_BEFORE_OPEN: std::sync::LazyLock<std::sync::Mutex<HashMap<PathBuf, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
@@ -2103,11 +2150,11 @@ fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + S
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
fn set_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
.lock()
|
||||
.expect("test publication hook lock should not be poisoned")
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
.insert((root.to_path_buf(), dst_volume.to_string(), dst_path.to_string()), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2135,6 +2182,7 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
||||
DELETE_VERSION_FAIL_AFTER_COMMIT
|
||||
.lock()
|
||||
@@ -2259,11 +2307,11 @@ fn run_inline_before_file_sync_admission(dst_path: &str) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_rename_data_after_first_publication(dst_path: &str) {
|
||||
fn run_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str) {
|
||||
let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
.lock()
|
||||
.expect("test publication hook lock should not be poisoned")
|
||||
.remove(dst_path);
|
||||
.remove(&(root.to_path_buf(), dst_volume.to_string(), dst_path.to_string()));
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
@@ -2361,9 +2409,6 @@ async fn remove_dst_base_before_commit(
|
||||
#[cfg(not(test))]
|
||||
fn run_inline_preparation_before_backup(_dst_path: &str) {}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn run_rename_data_after_first_publication(_dst_path: &str) {}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn should_fail_after_delete_data_staged(_path: &str) -> bool {
|
||||
false
|
||||
@@ -2443,6 +2488,10 @@ enum SyncMode {
|
||||
FileOnly,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||
)]
|
||||
struct FileCacheReclaimWriter {
|
||||
inner: File,
|
||||
reclaim_len: usize,
|
||||
@@ -2450,6 +2499,10 @@ struct FileCacheReclaimWriter {
|
||||
reclaimed: bool,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||
)]
|
||||
struct FileCacheReclaimReader {
|
||||
inner: File,
|
||||
reclaim_offset: u64,
|
||||
@@ -2515,6 +2568,10 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
|
||||
)]
|
||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
||||
// (three metric-key constructions) when general metrics are disabled.
|
||||
@@ -3067,6 +3124,7 @@ impl LocalIoBackend for StdBackend {
|
||||
use memmap2::MmapOptions;
|
||||
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
||||
|
||||
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
|
||||
struct MmapCopyReadResult {
|
||||
bytes: Bytes,
|
||||
access_check_duration: StdDuration,
|
||||
@@ -4700,6 +4758,10 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
||||
Arc::new(StdBackend::new(root))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
|
||||
)]
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
@@ -4751,6 +4813,25 @@ struct SnapshotLeaseEntry {
|
||||
tokens: HashSet<SnapshotLeaseToken>,
|
||||
pending_delete: Option<DeleteOptions>,
|
||||
deleting: bool,
|
||||
mutation_fence: Option<Arc<QuotaMutationFenceState>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct QuotaMutationFenceState {
|
||||
revoked: AtomicBool,
|
||||
running: AtomicUsize,
|
||||
notify: Notify,
|
||||
}
|
||||
|
||||
struct QuotaMutationFenceClaim {
|
||||
state: Arc<QuotaMutationFenceState>,
|
||||
}
|
||||
|
||||
impl Drop for QuotaMutationFenceClaim {
|
||||
fn drop(&mut self) {
|
||||
self.state.running.fetch_sub(1, Ordering::AcqRel);
|
||||
self.state.notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -5467,6 +5548,7 @@ impl LocalDisk {
|
||||
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
||||
let path_ref = path.as_ref();
|
||||
let path_str = path_ref.to_string_lossy();
|
||||
@@ -5544,15 +5626,24 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
// Check if a path is valid
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||
check_local_disk_valid_path(self.io_root(), path)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
||||
reject_local_disk_symlink_components(self.io_root(), path)
|
||||
}
|
||||
|
||||
// Batch path generation with single lock acquisition
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
||||
let mut results = Vec::with_capacity(requests.len());
|
||||
let mut cache_misses = Vec::new();
|
||||
@@ -6465,6 +6556,7 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||
Ok(f)
|
||||
@@ -7315,6 +7407,34 @@ fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
async fn claim_quota_mutation_fence(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
token: SnapshotLeaseToken,
|
||||
) -> Result<Arc<QuotaMutationFenceClaim>> {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: RUSTFS_META_BUCKET.to_string(),
|
||||
path: quota_mutation_fence_path(volume, path),
|
||||
};
|
||||
let state = {
|
||||
let registry = self.snapshot_leases.lock().await;
|
||||
let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?;
|
||||
let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?;
|
||||
if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
state.running.fetch_add(1, Ordering::AcqRel);
|
||||
Arc::clone(state)
|
||||
};
|
||||
if state.revoked.load(Ordering::Acquire) {
|
||||
state.running.fetch_sub(1, Ordering::AcqRel);
|
||||
state.notify.notify_waiters();
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
Ok(Arc::new(QuotaMutationFenceClaim { state }))
|
||||
}
|
||||
|
||||
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
|
||||
let path = format!("{object}/{data_dir}");
|
||||
let data_path = self.io_get_object_path(volume, &path)?;
|
||||
@@ -8638,17 +8758,41 @@ impl DiskAPI for LocalDisk {
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
mut fi: FileInfo,
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
let mut fi = fi;
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this lease establishes the local commit/delete order and
|
||||
// remains owned by any blocking syscall that outlives async cancellation.
|
||||
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
let quota_fence_token =
|
||||
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||
Some(value) => {
|
||||
let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?;
|
||||
Some(SnapshotLeaseToken::from_slice(token.as_bytes())?)
|
||||
}
|
||||
None if rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
) =>
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX);
|
||||
let quota_fence_claim = match quota_fence_token {
|
||||
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||
None => None,
|
||||
};
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
if fi.is_legacy_indexed_delete_marker() {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
@@ -8941,8 +9085,9 @@ impl DiskAPI for LocalDisk {
|
||||
.await?;
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path.is_some() {
|
||||
run_rename_data_after_first_publication(dst_path);
|
||||
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path);
|
||||
}
|
||||
|
||||
// Crash-consistency injection: hard power loss after the data dir
|
||||
@@ -9174,7 +9319,7 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
if let Some(src_file_path_parent) = src_file_path.parent() {
|
||||
if src_volume != super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
let _ = remove_std(src_file_path_parent);
|
||||
let _ = std::fs::remove_dir(src_file_path_parent);
|
||||
} else {
|
||||
let _ = self
|
||||
.delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
|
||||
@@ -9375,7 +9520,8 @@ impl DiskAPI for LocalDisk {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
run_rename_data_after_first_publication(dst_path);
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path);
|
||||
if sync {
|
||||
file_sync_admission = Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
@@ -9499,7 +9645,7 @@ impl DiskAPI for LocalDisk {
|
||||
if let Some(ref cleanup) = cleanup_path {
|
||||
let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
|
||||
} else if let Some(parent) = src_file_path.parent() {
|
||||
let _ = remove_std(parent);
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
// Heal reuses a version's `data_dir` and lands the rebuilt shard on
|
||||
@@ -9638,11 +9784,26 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
};
|
||||
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
let entry = registry.entries.entry(key).or_default();
|
||||
let state = entry
|
||||
.mutation_fence
|
||||
.get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default()));
|
||||
if state.revoked.load(Ordering::Acquire) {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
let token = SnapshotLeaseToken::new();
|
||||
entry.tokens.insert(token);
|
||||
return Ok(token);
|
||||
}
|
||||
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await;
|
||||
let token = {
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
if registry.entries.get(&key).is_some_and(|entry| entry.deleting) {
|
||||
@@ -9670,6 +9831,48 @@ impl DiskAPI for LocalDisk {
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
};
|
||||
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
|
||||
if !token.is_revoke_all() {
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||
return Ok(());
|
||||
};
|
||||
entry.tokens.remove(&token);
|
||||
let removable = entry.tokens.is_empty()
|
||||
&& entry
|
||||
.mutation_fence
|
||||
.as_ref()
|
||||
.is_none_or(|state| state.running.load(Ordering::Acquire) == 0);
|
||||
if removable {
|
||||
registry.entries.remove(&key);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let state = {
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(state) = entry.mutation_fence.as_ref().cloned() else {
|
||||
registry.entries.remove(&key);
|
||||
return Ok(());
|
||||
};
|
||||
state.revoked.store(true, Ordering::Release);
|
||||
entry.tokens.clear();
|
||||
state
|
||||
};
|
||||
loop {
|
||||
let notified = state.notify.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if state.running.load(Ordering::Acquire) == 0 {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
self.snapshot_leases.lock().await.entries.remove(&key);
|
||||
return Ok(());
|
||||
}
|
||||
let opts = {
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||
@@ -9840,6 +10043,12 @@ impl DiskAPI for LocalDisk {
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
crate::hp_guard!("LocalDisk::read_version");
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let metrics_path = if stage_metrics_enabled && crate::bucket::utils::is_meta_bucketname(volume) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
if !org_volume.is_empty() {
|
||||
let org_volume_path = self.io_get_bucket_path(org_volume)?;
|
||||
if !skip_access_checks(org_volume) {
|
||||
@@ -9849,36 +10058,46 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let path_resolve_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_PATH_RESOLVE, path_resolve_start);
|
||||
|
||||
let path_check_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_PATH_CHECK, path_check_start);
|
||||
|
||||
let read_data = opts.read_data;
|
||||
|
||||
let (data, _) = self
|
||||
.read_raw(volume, volume_dir.clone(), file_path, read_data)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e == DiskError::FileNotFound && !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
e
|
||||
}
|
||||
})?;
|
||||
let xlmeta_read_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let raw_read_result = self.read_raw(volume, volume_dir.clone(), file_path, read_data).await;
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_XLMETA_READ, xlmeta_read_start);
|
||||
let (data, _) = raw_read_result.map_err(|e| {
|
||||
if e == DiskError::FileNotFound && !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
e
|
||||
}
|
||||
})?;
|
||||
|
||||
let mut fi = get_file_info(
|
||||
&data,
|
||||
volume,
|
||||
path,
|
||||
version_id,
|
||||
FileInfoOpts {
|
||||
data: read_data,
|
||||
include_free_versions: opts.incl_free_versions,
|
||||
},
|
||||
)?;
|
||||
|
||||
fi.validate_for_metadata_read()?;
|
||||
let decode_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let file_info_result: Result<FileInfo> = (|| {
|
||||
let fi = get_file_info(
|
||||
&data,
|
||||
volume,
|
||||
path,
|
||||
version_id,
|
||||
FileInfoOpts {
|
||||
data: read_data,
|
||||
include_free_versions: opts.incl_free_versions,
|
||||
include_part_checksums: false,
|
||||
},
|
||||
)?;
|
||||
fi.validate_for_metadata_read()?;
|
||||
Ok(fi)
|
||||
})();
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_DECODE, decode_start);
|
||||
let mut fi = file_info_result?;
|
||||
if fi.is_canonical_delete_marker() {
|
||||
return Ok(fi);
|
||||
}
|
||||
@@ -10432,6 +10651,19 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
<Self as DiskAPI>::rename_data(self, src_volume, src_path, fi.clone(), dst_volume, dst_path).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_startup_cleanup_signal(
|
||||
startup_cleanup_ready: &AtomicU32,
|
||||
startup_cleanup_notify: &Notify,
|
||||
@@ -10561,6 +10793,108 @@ mod test {
|
||||
meta.marshal_msg().expect("test metadata should encode")
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn read_version_records_local_metadata_stage_breakdown() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should be created");
|
||||
let recorder = crate::test_metrics::CapturingRecorder::default();
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "bucket";
|
||||
let object = "stage-breakdown";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
|
||||
let object_dir = dir.path().join(bucket).join(object);
|
||||
fs::create_dir_all(&object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
test_meta(test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline")))),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
|
||||
disk.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read_version should succeed");
|
||||
|
||||
let meta_object = "stage-breakdown-meta";
|
||||
let meta_object_dir = dir.path().join(RUSTFS_META_BUCKET).join(meta_object);
|
||||
fs::create_dir_all(&meta_object_dir)
|
||||
.await
|
||||
.expect("internal metadata object directory should be created");
|
||||
fs::write(
|
||||
meta_object_dir.join(STORAGE_FORMAT_FILE),
|
||||
test_meta(test_file_info(meta_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"meta")))),
|
||||
)
|
||||
.await
|
||||
.expect("internal metadata should be written");
|
||||
|
||||
disk.read_version(
|
||||
"",
|
||||
RUSTFS_META_BUCKET,
|
||||
meta_object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("internal metadata read_version should succeed");
|
||||
});
|
||||
});
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
|
||||
|
||||
for stage in [
|
||||
GET_STAGE_READ_VERSION_PATH_RESOLVE,
|
||||
GET_STAGE_READ_VERSION_PATH_CHECK,
|
||||
GET_STAGE_READ_VERSION_XLMETA_READ,
|
||||
GET_STAGE_READ_VERSION_DECODE,
|
||||
] {
|
||||
assert_eq!(
|
||||
recorder
|
||||
.histogram_values(
|
||||
"rustfs_io_get_object_stage_duration_seconds",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("stage", stage)]
|
||||
)
|
||||
.len(),
|
||||
1,
|
||||
"{stage} should be recorded once for user-bucket LocalDisk::read_version"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder
|
||||
.histogram_values(
|
||||
"rustfs_io_get_object_stage_duration_seconds",
|
||||
&[("path", GET_OBJECT_PATH_INTERNAL_META), ("stage", stage)]
|
||||
)
|
||||
.len(),
|
||||
1,
|
||||
"{stage} should be recorded once for internal-meta LocalDisk::read_version"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_metadata_rollback_dir_avoids_real_data_dir_collision() {
|
||||
let target_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("version id should parse");
|
||||
@@ -12439,6 +12773,10 @@ mod test {
|
||||
.join(RUSTFS_META_TMP_BUCKET)
|
||||
.join(tmp_object)
|
||||
.join(new_data_dir.to_string());
|
||||
let tmp_parent = tmp_data_dir
|
||||
.parent()
|
||||
.expect("tmp data dir should have a parent")
|
||||
.to_path_buf();
|
||||
fs::create_dir_all(&tmp_data_dir)
|
||||
.await
|
||||
.expect("new tmp data dir should be created");
|
||||
@@ -12450,6 +12788,10 @@ mod test {
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
.await
|
||||
.expect("rename_data should commit");
|
||||
assert!(
|
||||
!tmp_parent.exists(),
|
||||
"successful non-inline commit should remove the empty staging parent"
|
||||
);
|
||||
|
||||
// The tmp xl.meta write point uses SyncMode::FileOnly: its parent dir
|
||||
// ({tmp}/{tmp_object}) must not be fsynced.
|
||||
@@ -12654,6 +12996,9 @@ mod test {
|
||||
let tmp_object = "tmp-new-inline";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
let tmp_parent = disk
|
||||
.get_object_path(RUSTFS_META_TMP_BUCKET, tmp_object)
|
||||
.expect("tmp parent should resolve");
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let version_id = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("version id should parse");
|
||||
@@ -12662,6 +13007,7 @@ mod test {
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
.await
|
||||
.expect("inline rename_data should commit the new object");
|
||||
assert!(!tmp_parent.exists(), "successful inline commit should remove the empty staging parent");
|
||||
|
||||
let bucket_dir = disk.get_bucket_path(bucket).expect("bucket path should resolve");
|
||||
let prefix_dir = disk.get_object_path(bucket, "prefix").expect("prefix path should resolve");
|
||||
@@ -12685,6 +13031,34 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_data_inline_preserves_non_empty_staging_parent() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "inline-staging-sentinel-bucket";
|
||||
let object = "inline-object";
|
||||
let tmp_object = "inline-stage-with-sentinel";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let tmp_parent = disk
|
||||
.get_object_path(RUSTFS_META_TMP_BUCKET, tmp_object)
|
||||
.expect("tmp parent should resolve");
|
||||
fs::create_dir_all(&tmp_parent).await.expect("tmp parent should be created");
|
||||
let sentinel = tmp_parent.join("sentinel");
|
||||
fs::write(&sentinel, b"keep").await.expect("sentinel should be written");
|
||||
|
||||
let fi = test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline-payload")));
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
|
||||
.await
|
||||
.expect("non-empty staging cleanup must not negate the committed object");
|
||||
|
||||
assert_eq!(fs::read(&sentinel).await.expect("sentinel should remain"), b"keep");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
@@ -12844,7 +13218,7 @@ mod test {
|
||||
let replacement_staging_parent_for_hook = replacement_staging_parent.clone();
|
||||
let staged_metadata_for_hook = staged_metadata.clone();
|
||||
let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
set_rename_data_after_first_publication(&disk.root, bucket, object, move || {
|
||||
std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook)
|
||||
.expect_err("the destination object identity must remain pinned until xl.meta commits");
|
||||
std::fs::rename(&staging_parent_for_hook, &replacement_staging_parent_for_hook)
|
||||
@@ -12859,7 +13233,10 @@ mod test {
|
||||
.expect("non-inline rename_data should commit");
|
||||
|
||||
assert!(!replacement_dir.exists(), "the destination object directory must not be replaced");
|
||||
assert!(staging_parent.exists(), "the guarded staging parent must retain its identity");
|
||||
assert!(
|
||||
!staging_parent.exists(),
|
||||
"successful commit should remove the empty staging parent after releasing its guard"
|
||||
);
|
||||
assert!(
|
||||
!replacement_staging_parent.exists(),
|
||||
"the staging parent must not be replaced between data and metadata publication"
|
||||
@@ -13108,7 +13485,7 @@ mod test {
|
||||
let replacement_dir_for_hook = replacement_dir.clone();
|
||||
let staged_metadata_for_hook = staged_metadata.clone();
|
||||
let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
set_rename_data_after_first_publication(&disk.root, bucket, object, move || {
|
||||
std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook)
|
||||
.expect_err("the destination object identity must remain pinned after publishing its rollback backup");
|
||||
std::fs::rename(&staged_metadata_for_hook, &replacement_staged_metadata_for_hook)
|
||||
@@ -13474,7 +13851,7 @@ mod test {
|
||||
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
set_rename_data_after_first_publication(&disk.root, bucket, object, move || {
|
||||
entered_tx.send(()).expect("signal first publication");
|
||||
release_rx.recv().expect("wait while delete_volume is blocked");
|
||||
});
|
||||
@@ -14068,7 +14445,7 @@ mod test {
|
||||
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
set_rename_data_after_first_publication(&disk.root, bucket, object, move || {
|
||||
published_tx.send(()).expect("signal backup publication");
|
||||
release_rx.recv().expect("wait for lock-order assertion");
|
||||
});
|
||||
@@ -18723,6 +19100,48 @@ mod test {
|
||||
assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let root_dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let bucket = "quota-fence-volume";
|
||||
let object = "object";
|
||||
let fence_path = quota_mutation_fence_path(bucket, object);
|
||||
let token = disk
|
||||
.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path)
|
||||
.await
|
||||
.expect("quota mutation token should be prepared");
|
||||
let claim = disk
|
||||
.claim_quota_mutation_fence(bucket, object, token)
|
||||
.await
|
||||
.expect("prepared token should be claimable");
|
||||
|
||||
let release_disk = Arc::clone(&disk);
|
||||
let mut release = tokio::spawn(async move {
|
||||
release_disk
|
||||
.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all())
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(),
|
||||
"revoke must wait until an already claimed mutation has finished"
|
||||
);
|
||||
|
||||
drop(claim);
|
||||
tokio::time::timeout(Duration::from_secs(1), release)
|
||||
.await
|
||||
.expect("revoke should wake after the final claim drops")
|
||||
.expect("revoke task should not panic")
|
||||
.expect("revoke should succeed");
|
||||
assert!(matches!(
|
||||
disk.claim_quota_mutation_fence(bucket, object, token).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_version_keeps_later_part_until_snapshot_release() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod disk_store;
|
||||
pub mod endpoint;
|
||||
@@ -55,6 +54,7 @@ pub fn part_transaction_path(part_path: &str) -> String {
|
||||
|
||||
use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::disk_store::LocalDiskWrapper;
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
use crate::disk::local::ScanGuard;
|
||||
@@ -65,12 +65,35 @@ use error::{Error, Result};
|
||||
use local::LocalDisk;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
use rustfs_rio::ChunkReaderBox;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
|
||||
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||
|
||||
pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let mut input = Vec::with_capacity(bucket.len() + object.len() + 1);
|
||||
input.extend_from_slice(bucket.as_bytes());
|
||||
input.push(0);
|
||||
input.extend_from_slice(object.as_bytes());
|
||||
let digest = Sha256::digest(input);
|
||||
format!(
|
||||
"{QUOTA_MUTATION_FENCE_PREFIX}{}",
|
||||
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool {
|
||||
path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX)
|
||||
.is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
}
|
||||
|
||||
pub type DiskStore = Arc<Disk>;
|
||||
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
@@ -95,6 +118,20 @@ impl SnapshotLeaseToken {
|
||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||
self.0.as_bytes()
|
||||
}
|
||||
|
||||
pub(crate) fn as_uuid(self) -> Uuid {
|
||||
self.0
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn revoke_all() -> Self {
|
||||
Self(Uuid::nil())
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn is_revoke_all(self) -> bool {
|
||||
self.0.is_nil()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SnapshotLeaseToken {
|
||||
@@ -397,10 +434,8 @@ impl DiskAPI for Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
|
||||
}
|
||||
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -427,6 +462,19 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<ChunkReaderBox>> {
|
||||
match self {
|
||||
Disk::Local(_) => Ok(None),
|
||||
Disk::Remote(remote_disk) => remote_disk.read_file_stream_chunks(volume, path, offset, length).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes> {
|
||||
match self {
|
||||
@@ -617,6 +665,30 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
match self {
|
||||
@@ -865,6 +937,18 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader>;
|
||||
|
||||
/// Returns an owned-chunk stream when the backing transport can preserve
|
||||
/// receive-buffer ownership. `None` retains the ordinary reader path.
|
||||
async fn read_file_stream_chunks(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_path: &str,
|
||||
_offset: usize,
|
||||
_length: usize,
|
||||
) -> Result<Option<ChunkReaderBox>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// File read using mmap-then-copy on Unix or an efficient read on non-Unix.
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes>;
|
||||
|
||||
@@ -1029,6 +1113,10 @@ pub struct DiskInfo {
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
|
||||
)]
|
||||
pub struct Info {
|
||||
pub total: u64,
|
||||
pub free: u64,
|
||||
@@ -1287,6 +1375,7 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
||||
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
||||
}
|
||||
|
||||
@@ -306,12 +306,20 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
|
||||
pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
|
||||
*self.external_guard.lock() = Some(guard);
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -327,6 +335,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: namespace_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
external_guard: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -562,6 +571,10 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
|
||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||
/// itself.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||
)]
|
||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||
let entry = entry?;
|
||||
@@ -574,6 +587,7 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||
/// concurrently, bounded both per directory and process-wide.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
||||
}
|
||||
@@ -1800,10 +1814,6 @@ impl RenameCommitGuard {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
|
||||
self.destination_directory_guard(directory, false)
|
||||
}
|
||||
|
||||
pub(crate) fn create_destination_directory_for_path_access(
|
||||
&self,
|
||||
directory: &Path,
|
||||
@@ -2849,13 +2859,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if a file exists.
|
||||
/// Returns true if the file exists, false otherwise.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn file_exists(path: impl AsRef<Path>) -> bool {
|
||||
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether an [`io::Error`] means "the directory is not empty".
|
||||
///
|
||||
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
||||
|
||||
@@ -26,6 +26,7 @@ pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE: &str = "skip_data_c
|
||||
pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_EMPTY_PAYLOAD: &str = "skip_empty_payload";
|
||||
|
||||
pub(crate) trait DecodeWorkspace: Send + Sync + 'static {
|
||||
#[allow(dead_code, reason = "workspace width asserted by decode_reader tests (backlog#1823)")]
|
||||
fn shard_len(&self) -> usize;
|
||||
}
|
||||
|
||||
@@ -33,11 +34,14 @@ pub(crate) trait ErasureDecodeEngine: Send + Sync + 'static {
|
||||
type Workspace: DecodeWorkspace;
|
||||
|
||||
fn data_shards(&self) -> usize;
|
||||
#[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")]
|
||||
fn parity_shards(&self) -> usize;
|
||||
fn block_size(&self) -> usize;
|
||||
fn engine_name(&self) -> &'static str;
|
||||
|
||||
#[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")]
|
||||
fn supports_progressive_decode(&self) -> bool;
|
||||
#[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")]
|
||||
fn supports_aligned_shards(&self) -> bool;
|
||||
|
||||
fn prepare_workspace(&self, shard_len: usize) -> io::Result<Self::Workspace>;
|
||||
|
||||
@@ -24,6 +24,7 @@ impl RustfsCodecDecodeWorkspace {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(dead_code, reason = "workspace width asserted by decode_reader tests (backlog#1823)")]
|
||||
pub(crate) fn shard_len(&self) -> usize {
|
||||
self.shard_len
|
||||
}
|
||||
@@ -76,6 +77,13 @@ impl ShardBufferPool {
|
||||
self.buffers[index] = Some(buf);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn stored_allocation(&self, index: usize) -> Option<(*const u8, usize)> {
|
||||
self.buffers
|
||||
.get(index)
|
||||
.and_then(|buf| buf.as_ref().map(|buf| (buf.as_ptr(), buf.capacity())))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn stored_capacity(&self, index: usize) -> Option<usize> {
|
||||
self.buffers.get(index).and_then(|buf| buf.as_ref().map(Vec::capacity))
|
||||
|
||||
@@ -14,7 +14,10 @@
|
||||
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::future::poll_fn;
|
||||
use std::io::IoSlice;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tracing::error;
|
||||
@@ -23,6 +26,18 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
|
||||
const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
|
||||
const MAX_RETAINED_CHUNKS_PER_BLOCK: usize = 64;
|
||||
const MAX_CHUNK_POLLS_PER_YIELD: usize = MAX_RETAINED_CHUNKS_PER_BLOCK + 1;
|
||||
|
||||
/// Result of polling an optional owned-chunk handoff.
|
||||
pub enum ShardChunkRead {
|
||||
/// The source does not support owned-chunk handoff and remains untouched.
|
||||
Unsupported,
|
||||
/// The source reached EOF.
|
||||
Eof,
|
||||
/// A non-empty chunk containing at most the requested number of bytes.
|
||||
Chunk(bytes::Bytes),
|
||||
}
|
||||
|
||||
/// A shard source that may already hold its bytes in memory.
|
||||
///
|
||||
@@ -42,6 +57,12 @@ pub trait ShardSource: AsyncRead + Send + Sync + Unpin {
|
||||
fn try_take_block(&mut self, _n: usize) -> Option<bytes::Bytes> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Polls one owned chunk when the source supports chunk handoff.
|
||||
/// `Unsupported` must leave the source untouched.
|
||||
fn poll_read_chunk(self: Pin<&mut Self>, _cx: &mut Context<'_>, _max: usize) -> Poll<std::io::Result<ShardChunkRead>> {
|
||||
Poll::Ready(Ok(ShardChunkRead::Unsupported))
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrowed and owned byte slices are ordinary streaming sources: they carry no
|
||||
@@ -75,6 +96,9 @@ pin_project! {
|
||||
// contiguous on-disk `[hash][data]` block so both are pulled in a single
|
||||
// pass; grown lazily and never shrunk.
|
||||
buf: Vec<u8>,
|
||||
// Reused owned chunk vector for the remote HTTP fast path. Keeping the
|
||||
// allocation with the reader avoids allocating once per bitrot block.
|
||||
chunks: Vec<bytes::Bytes>,
|
||||
skip_verify: bool,
|
||||
last_verify_duration: Duration,
|
||||
}
|
||||
@@ -91,6 +115,7 @@ where
|
||||
hash_algo: algo,
|
||||
shard_size,
|
||||
buf: Vec::new(),
|
||||
chunks: Vec::new(),
|
||||
skip_verify,
|
||||
last_verify_duration: Duration::ZERO,
|
||||
}
|
||||
@@ -100,6 +125,11 @@ where
|
||||
self.last_verify_duration
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn inner_ref(&self) -> &R {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
/// Read a single (hash+data) block, verify hash, and copy `out.len()` bytes
|
||||
/// into `out`. Returns an error if the shard is short, the hash mismatches,
|
||||
/// or `out` is larger than one shard. On error `out`'s contents are
|
||||
@@ -260,11 +290,6 @@ where
|
||||
|
||||
let need = hash_size + want;
|
||||
|
||||
// In-memory fast path: the block is already resident, so slice it instead
|
||||
// of copying it into the scratch buffer first (rustfs/backlog#1159). One
|
||||
// copy (`extend_from_slice`) instead of two. A source that cannot serve
|
||||
// `need` bytes returns `None` and falls through to the scratch path,
|
||||
// keeping the short-read contract.
|
||||
if let Some(block) = self.inner.try_take_block(need) {
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block)?;
|
||||
out.extend_from_slice(data);
|
||||
@@ -272,6 +297,126 @@ where
|
||||
return Ok(want);
|
||||
}
|
||||
|
||||
self.chunks.clear();
|
||||
let handed_off = {
|
||||
let inner = &mut self.inner;
|
||||
let chunks = &mut self.chunks;
|
||||
let tail_buf = &mut self.buf;
|
||||
let mut received = 0usize;
|
||||
poll_fn(|cx| {
|
||||
for _ in 0..MAX_CHUNK_POLLS_PER_YIELD {
|
||||
let next = match Pin::new(&mut *inner).poll_read_chunk(cx, need - received) {
|
||||
Poll::Ready(Ok(next)) => next,
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
};
|
||||
let chunk = match next {
|
||||
ShardChunkRead::Unsupported if received == 0 => return Poll::Ready(Ok(false)),
|
||||
ShardChunkRead::Unsupported => {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"chunk handoff became unavailable after transferring data",
|
||||
)));
|
||||
}
|
||||
ShardChunkRead::Eof => {
|
||||
return Poll::Ready(Err(short_shard_read(received.saturating_sub(hash_size), want)));
|
||||
}
|
||||
ShardChunkRead::Chunk(chunk) => chunk,
|
||||
};
|
||||
|
||||
if received == 0 {
|
||||
tail_buf.clear();
|
||||
}
|
||||
if chunk.is_empty() {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"chunk handoff returned an empty chunk",
|
||||
)));
|
||||
}
|
||||
let remaining = need - received;
|
||||
if chunk.len() > remaining {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"chunk handoff exceeded its requested boundary",
|
||||
)));
|
||||
}
|
||||
received += chunk.len();
|
||||
|
||||
if chunks.len() == MAX_RETAINED_CHUNKS_PER_BLOCK {
|
||||
if tail_buf.is_empty() {
|
||||
tail_buf.reserve_exact(need - (received - chunk.len()));
|
||||
}
|
||||
tail_buf.extend_from_slice(&chunk);
|
||||
} else {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
if received == need {
|
||||
return Poll::Ready(Ok(true));
|
||||
}
|
||||
}
|
||||
cx.waker().wake_by_ref();
|
||||
Poll::Pending
|
||||
})
|
||||
.await?
|
||||
};
|
||||
if handed_off {
|
||||
if self.chunks.len() == 1 && self.buf.is_empty() {
|
||||
let block = &self.chunks[0];
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, block)?;
|
||||
out.extend_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
return Ok(want);
|
||||
}
|
||||
|
||||
let block_chunks = || {
|
||||
self.chunks
|
||||
.iter()
|
||||
.map(|chunk| chunk.as_ref())
|
||||
.chain((!self.buf.is_empty()).then_some(self.buf.as_slice()))
|
||||
};
|
||||
if !self.skip_verify {
|
||||
let verify_start = std::time::Instant::now();
|
||||
let actual_hash = self
|
||||
.hash_algo
|
||||
.hash_encode_slices(block_chunks().scan(hash_size, |skip, chunk| {
|
||||
let start = (*skip).min(chunk.len());
|
||||
*skip -= start;
|
||||
Some(&chunk[start..])
|
||||
}));
|
||||
let verify = verify_start.elapsed();
|
||||
let mut hash_offset = 0;
|
||||
let mut remaining = hash_size;
|
||||
for chunk in block_chunks() {
|
||||
let take = remaining.min(chunk.len());
|
||||
if actual_hash.as_ref()[hash_offset..hash_offset + take] != chunk[..take] {
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len = want,
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
hash_offset += take;
|
||||
remaining -= take;
|
||||
if remaining == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.last_verify_duration = verify;
|
||||
}
|
||||
let mut skip = hash_size;
|
||||
for chunk in block_chunks() {
|
||||
let start = skip.min(chunk.len());
|
||||
skip -= start;
|
||||
out.extend_from_slice(&chunk[start..]);
|
||||
}
|
||||
return Ok(want);
|
||||
}
|
||||
|
||||
// Streaming path: same single pass and same verification as `read`; only
|
||||
// the sink differs (`extend_from_slice` into `out` instead of
|
||||
// `copy_from_slice` into a pre-zeroed buffer).
|
||||
@@ -677,18 +822,167 @@ impl BitrotWriterWrapper {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ShardSource;
|
||||
use super::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
};
|
||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::io::{Cursor, IoSlice};
|
||||
use std::collections::VecDeque;
|
||||
use std::io::{self, Cursor, IoSlice};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
|
||||
struct FragmentedSource {
|
||||
chunks: VecDeque<Bytes>,
|
||||
}
|
||||
|
||||
impl FragmentedSource {
|
||||
fn new(bytes: Vec<u8>, fragment_sizes: &[usize]) -> Self {
|
||||
let mut chunks = VecDeque::new();
|
||||
let mut offset = 0;
|
||||
for &size in fragment_sizes {
|
||||
let end = (offset + size).min(bytes.len());
|
||||
if offset < end {
|
||||
chunks.push_back(Bytes::copy_from_slice(&bytes[offset..end]));
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
if offset < bytes.len() {
|
||||
chunks.push_back(Bytes::copy_from_slice(&bytes[offset..]));
|
||||
}
|
||||
Self { chunks }
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for FragmentedSource {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(io::Error::other("fragmented source must use chunk handoff")))
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardSource for FragmentedSource {
|
||||
fn poll_read_chunk(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, max: usize) -> Poll<io::Result<ShardChunkRead>> {
|
||||
let Some(mut chunk) = self.chunks.pop_front() else {
|
||||
return Poll::Ready(Ok(ShardChunkRead::Eof));
|
||||
};
|
||||
if chunk.len() > max {
|
||||
self.chunks.push_front(chunk.split_off(max));
|
||||
chunk.truncate(max);
|
||||
}
|
||||
Poll::Ready(Ok(ShardChunkRead::Chunk(chunk)))
|
||||
}
|
||||
}
|
||||
|
||||
struct GeneratedChunkSource {
|
||||
bytes: Bytes,
|
||||
offset: usize,
|
||||
fragment_size: usize,
|
||||
fail_at: Option<usize>,
|
||||
}
|
||||
|
||||
impl GeneratedChunkSource {
|
||||
fn new(bytes: Vec<u8>, fragment_size: usize) -> Self {
|
||||
assert!(fragment_size > 0);
|
||||
Self {
|
||||
bytes: Bytes::from(bytes),
|
||||
offset: 0,
|
||||
fragment_size,
|
||||
fail_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn failing(bytes: Vec<u8>, fragment_size: usize, fail_at: usize) -> Self {
|
||||
Self {
|
||||
fail_at: Some(fail_at),
|
||||
..Self::new(bytes, fragment_size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for GeneratedChunkSource {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(io::Error::other("generated source must use chunk handoff")))
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardSource for GeneratedChunkSource {
|
||||
fn poll_read_chunk(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, max: usize) -> Poll<io::Result<ShardChunkRead>> {
|
||||
if self.fail_at == Some(self.offset) {
|
||||
return Poll::Ready(Err(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::BodyStreamAborted,
|
||||
)));
|
||||
}
|
||||
if self.offset == self.bytes.len() {
|
||||
return Poll::Ready(Ok(ShardChunkRead::Eof));
|
||||
}
|
||||
let error_limit = self.fail_at.unwrap_or(self.bytes.len());
|
||||
let take = self
|
||||
.fragment_size
|
||||
.min(max)
|
||||
.min(error_limit - self.offset)
|
||||
.min(self.bytes.len() - self.offset);
|
||||
let start = self.offset;
|
||||
self.offset += take;
|
||||
Poll::Ready(Ok(ShardChunkRead::Chunk(self.bytes.slice(start..start + take))))
|
||||
}
|
||||
}
|
||||
|
||||
struct InvalidChunkSource {
|
||||
mode: InvalidChunkMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum InvalidChunkMode {
|
||||
Empty,
|
||||
Oversized,
|
||||
UnsupportedAfterChunk,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl AsyncRead for InvalidChunkSource {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(io::Error::other("invalid source must use chunk handoff")))
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardSource for InvalidChunkSource {
|
||||
fn poll_read_chunk(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, max: usize) -> Poll<io::Result<ShardChunkRead>> {
|
||||
match self.mode {
|
||||
InvalidChunkMode::Empty => Poll::Ready(Ok(ShardChunkRead::Chunk(Bytes::new()))),
|
||||
InvalidChunkMode::Oversized => Poll::Ready(Ok(ShardChunkRead::Chunk(Bytes::from(vec![0; max + 1])))),
|
||||
InvalidChunkMode::UnsupportedAfterChunk => {
|
||||
self.mode = InvalidChunkMode::Unsupported;
|
||||
Poll::Ready(Ok(ShardChunkRead::Chunk(Bytes::from_static(b"x"))))
|
||||
}
|
||||
InvalidChunkMode::Unsupported => Poll::Ready(Ok(ShardChunkRead::Unsupported)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ScratchReuseSource {
|
||||
block: Option<Bytes>,
|
||||
saw_reused_scratch: bool,
|
||||
}
|
||||
|
||||
impl AsyncRead for ScratchReuseSource {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let Some(block) = self.block.take() else {
|
||||
return Poll::Ready(Ok(()));
|
||||
};
|
||||
self.saw_reused_scratch = buf.initialize_unfilled()[..block.len()].iter().all(|byte| *byte == 0xa5);
|
||||
buf.put_slice(&block);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardSource for ScratchReuseSource {}
|
||||
|
||||
#[derive(Default)]
|
||||
struct VectoredCountingWriter {
|
||||
@@ -1446,6 +1740,70 @@ mod tests {
|
||||
assert!(out.is_empty(), "corrupt bytes must never reach the caller's buffer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_verifies_data_split_across_hash_boundaries() {
|
||||
const SHARD: usize = 4096;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let data: Vec<u8> = (0..SHARD).map(|index| (index % 251) as u8).collect();
|
||||
let mut encoded = Vec::new();
|
||||
BitrotWriter::new(&mut encoded, SHARD, algo.clone())
|
||||
.write(&data)
|
||||
.await
|
||||
.expect("write shard");
|
||||
|
||||
let mut output = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(FragmentedSource::new(encoded, &[3, 11, 19, 37, 128]), SHARD, algo, false)
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect("fragmented shard must verify");
|
||||
|
||||
assert_eq!(output, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_never_appends_a_corrupt_shard() {
|
||||
const SHARD: usize = 4096;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let mut encoded = Vec::new();
|
||||
BitrotWriter::new(&mut encoded, SHARD, algo.clone())
|
||||
.write(&vec![9u8; SHARD])
|
||||
.await
|
||||
.expect("write shard");
|
||||
let last = encoded.len() - 1;
|
||||
encoded[last] ^= 0xff;
|
||||
|
||||
let mut output = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(FragmentedSource::new(encoded, &[7, 17, 31]), SHARD, algo, false)
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect_err("corrupt fragmented shard must fail");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_does_not_hash_when_verification_is_skipped() {
|
||||
const SHARD: usize = 4096;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let mut encoded = Vec::new();
|
||||
BitrotWriter::new(&mut encoded, SHARD, algo.clone())
|
||||
.write(&vec![9u8; SHARD])
|
||||
.await
|
||||
.expect("write shard");
|
||||
encoded[0] ^= 0xff;
|
||||
|
||||
let mut output = Vec::with_capacity(SHARD);
|
||||
let mut reader = BitrotReader::new(FragmentedSource::new(encoded, &[7, 17, 31]), SHARD, algo, true);
|
||||
reader
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect("skipped verification must accept fragmented shard bytes");
|
||||
|
||||
assert_eq!(reader.last_verify_duration(), Duration::ZERO);
|
||||
assert_eq!(output, vec![9u8; SHARD]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_appending_rejects_a_want_larger_than_the_shard() {
|
||||
let algo = HashAlgorithm::HighwayHash256;
|
||||
@@ -1497,10 +1855,21 @@ mod tests {
|
||||
|
||||
// Equivalence: same bytes out of both paths.
|
||||
let mut via_mem: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(Cursor::new(Bytes::from(encoded.clone())), SHARD, algo.clone(), false)
|
||||
let mut memory_reader = BitrotReader::new(Cursor::new(Bytes::from(encoded.clone())), SHARD, algo.clone(), false);
|
||||
memory_reader
|
||||
.read_appending(&mut via_mem, SHARD)
|
||||
.await
|
||||
.expect("in-memory read");
|
||||
assert_eq!(
|
||||
memory_reader.chunks.capacity(),
|
||||
0,
|
||||
"the synchronous fast path must not allocate chunk storage"
|
||||
);
|
||||
assert_eq!(
|
||||
memory_reader.buf.capacity(),
|
||||
0,
|
||||
"the synchronous fast path must not allocate scratch storage"
|
||||
);
|
||||
|
||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
@@ -1537,4 +1906,152 @@ mod tests {
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(out.is_empty(), "corrupt bytes must never reach the caller's buffer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_fallback_reuses_initialized_scratch() {
|
||||
const SHARD: usize = 4096;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let data = vec![7u8; SHARD];
|
||||
let encoded = encode_one_block(&data, SHARD, algo.clone()).await;
|
||||
let source = ScratchReuseSource {
|
||||
block: Some(Bytes::copy_from_slice(&encoded)),
|
||||
saw_reused_scratch: false,
|
||||
};
|
||||
let mut reader = BitrotReader::new(source, SHARD, algo, false);
|
||||
reader.buf = vec![0xa5; encoded.len()];
|
||||
let mut output = Vec::new();
|
||||
|
||||
reader
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect("streaming fallback should verify");
|
||||
|
||||
assert!(reader.inner.saw_reused_scratch, "capability probing must not clear reusable scratch");
|
||||
assert_eq!(output, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_bounds_production_sized_one_byte_fragments() {
|
||||
const SHARD: usize = 1024 * 1024 / 4;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let data: Vec<u8> = (0..SHARD).map(|index| (index % 251) as u8).collect();
|
||||
let encoded = encode_one_block(&data, SHARD, algo.clone()).await;
|
||||
let encoded_len = encoded.len();
|
||||
let mut reader = BitrotReader::new(GeneratedChunkSource::new(encoded, 1), SHARD, algo, false);
|
||||
let mut output = Vec::with_capacity(SHARD);
|
||||
|
||||
reader
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect("one-byte fragments should verify with bounded retained state");
|
||||
|
||||
assert_eq!(output, data);
|
||||
assert_eq!(reader.chunks.len(), MAX_RETAINED_CHUNKS_PER_BLOCK);
|
||||
assert!(reader.chunks.capacity() <= MAX_RETAINED_CHUNKS_PER_BLOCK);
|
||||
assert_eq!(reader.buf.len(), encoded_len - MAX_RETAINED_CHUNKS_PER_BLOCK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_keeps_sixty_four_frames_zero_copy_and_respects_poll_budget() {
|
||||
const SHARD: usize = 1024 * 1024;
|
||||
const FRAME: usize = 16 * 1024;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
|
||||
let small_data = vec![3u8; 4096];
|
||||
let small_encoded = encode_one_block(&small_data, 4096, algo.clone()).await;
|
||||
let mut exact_reader =
|
||||
BitrotReader::new(FragmentedSource::new(small_encoded.clone(), &[1; 63]), 4096, algo.clone(), false);
|
||||
let mut exact_output = Vec::new();
|
||||
exact_reader
|
||||
.read_appending(&mut exact_output, 4096)
|
||||
.await
|
||||
.expect("exactly sixty-four frames should verify");
|
||||
assert_eq!(exact_output, small_data);
|
||||
assert_eq!(exact_reader.chunks.len(), MAX_RETAINED_CHUNKS_PER_BLOCK);
|
||||
assert!(exact_reader.buf.is_empty(), "the threshold itself must remain zero-copy");
|
||||
|
||||
let mut yielded_reader = BitrotReader::new(FragmentedSource::new(small_encoded, &[1; 65]), 4096, algo.clone(), false);
|
||||
let mut yielded_output = Vec::new();
|
||||
let mut yielded_read = Box::pin(yielded_reader.read_appending(&mut yielded_output, 4096));
|
||||
let mut cx = Context::from_waker(std::task::Waker::noop());
|
||||
assert!(std::future::Future::poll(yielded_read.as_mut(), &mut cx).is_pending());
|
||||
assert!(matches!(std::future::Future::poll(yielded_read.as_mut(), &mut cx), Poll::Ready(Ok(4096))));
|
||||
drop(yielded_read);
|
||||
assert_eq!(yielded_output, small_data);
|
||||
|
||||
let data = vec![7u8; SHARD];
|
||||
let encoded = encode_one_block(&data, SHARD, algo.clone()).await;
|
||||
let mut reader = BitrotReader::new(FragmentedSource::new(encoded, &[FRAME; 64]), SHARD, algo, false);
|
||||
let mut output = Vec::with_capacity(SHARD);
|
||||
let mut read = Box::pin(reader.read_appending(&mut output, SHARD));
|
||||
assert!(
|
||||
matches!(std::future::Future::poll(read.as_mut(), &mut cx), Poll::Ready(Ok(SHARD))),
|
||||
"sixty-five normal HTTP frames should complete without a cooperative yield"
|
||||
);
|
||||
drop(read);
|
||||
|
||||
assert_eq!(output, data);
|
||||
assert_eq!(reader.chunks.len(), MAX_RETAINED_CHUNKS_PER_BLOCK);
|
||||
assert_eq!(reader.buf.len(), HashAlgorithm::HighwayHash256S.size());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_tail_failures_preserve_errors_and_output() {
|
||||
const SHARD: usize = 4096;
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let data = vec![7u8; SHARD];
|
||||
let encoded = encode_one_block(&data, SHARD, algo.clone()).await;
|
||||
let sentinel = vec![1u8, 2, 3];
|
||||
|
||||
let mut short_output = sentinel.clone();
|
||||
let short_err = BitrotReader::new(GeneratedChunkSource::new(encoded[..100].to_vec(), 1), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut short_output, SHARD)
|
||||
.await
|
||||
.expect_err("EOF after the retention threshold must stay a short read");
|
||||
assert_eq!(short_err.kind(), io::ErrorKind::UnexpectedEof);
|
||||
assert_eq!(short_output, sentinel);
|
||||
|
||||
let mut corrupt = encoded.clone();
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0xff;
|
||||
let mut corrupt_output = sentinel.clone();
|
||||
let corrupt_err = BitrotReader::new(GeneratedChunkSource::new(corrupt, 1), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut corrupt_output, SHARD)
|
||||
.await
|
||||
.expect_err("corrupt coalesced tail must fail verification");
|
||||
assert_eq!(corrupt_err.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(corrupt_output, sentinel);
|
||||
|
||||
let mut failed_output = sentinel.clone();
|
||||
let body_err = BitrotReader::new(GeneratedChunkSource::failing(encoded, 1, 65), SHARD, algo, false)
|
||||
.read_appending(&mut failed_output, SHARD)
|
||||
.await
|
||||
.expect_err("a terminal body error must not become EOF");
|
||||
let source = body_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.expect("body error should retain internode classification");
|
||||
assert_eq!(source.kind(), rustfs_rio::InternodeHttpErrorKind::BodyStreamAborted);
|
||||
assert_eq!(failed_output, sentinel);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunked_handoff_rejects_invalid_source_contracts() {
|
||||
const SHARD: usize = 64;
|
||||
for mode in [
|
||||
InvalidChunkMode::Empty,
|
||||
InvalidChunkMode::Oversized,
|
||||
InvalidChunkMode::UnsupportedAfterChunk,
|
||||
] {
|
||||
let source = InvalidChunkSource { mode };
|
||||
let mut output = vec![9u8];
|
||||
let err = BitrotReader::new(source, SHARD, HashAlgorithm::HighwayHash256S, false)
|
||||
.read_appending(&mut output, SHARD)
|
||||
.await
|
||||
.expect_err("invalid chunk contracts must fail closed");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(output, vec![9u8]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@ use crate::disk::error_reduce::reduce_errs;
|
||||
use crate::erasure::codec::workspace::ShardBufferPool;
|
||||
use crate::erasure::coding::{BitrotReader, Erasure};
|
||||
use crate::io_support::bitrot::DeferredReaderStripeHandle;
|
||||
use crate::set_disk::shard_source::{ShardReadCost, ShardStripeSource, StripeReadState};
|
||||
use crate::set_disk::shard_source::{
|
||||
INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardStripeSource, StripeReadState,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use pin_project_lite::pin_project;
|
||||
@@ -41,9 +43,6 @@ use tracing::{debug, error, warn};
|
||||
|
||||
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
|
||||
|
||||
const INLINE_SHARD_SLOTS: usize = 32;
|
||||
type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
|
||||
@@ -214,6 +213,7 @@ fn shard_read_launch_rank(cost: ShardReadCost) -> u8 {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "launch ordering asserted by this file's tests (backlog#1823)")]
|
||||
fn shard_read_launch_order(read_costs: &[ShardReadCost], num_readers: usize, locality_preference_enabled: bool) -> Vec<usize> {
|
||||
let mut order: Vec<usize> = (0..num_readers).collect();
|
||||
if locality_preference_enabled {
|
||||
@@ -392,6 +392,7 @@ pub(crate) struct ParallelReader<R> {
|
||||
// Request-scoped shard buffers keyed by shard index. Keeping ownership in
|
||||
// `ParallelReader` avoids dropping unused parity/backup slot buffers between stripes.
|
||||
buffers: ShardBufferPool,
|
||||
stripe_state: Option<Box<StripeReadState>>,
|
||||
// Lockstep-path state (verify_reconstruction == true). `engaged[i]` marks
|
||||
// readers that participate in each stripe read: all data slots from the
|
||||
// start, parity slots only once a data shard is missing/dead. Unengaged
|
||||
@@ -408,6 +409,10 @@ where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
// Readers should handle disk errors before being passed in, ensuring each reader reaches the available number of BitrotReaders
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "ParallelReader constructor used only by this file's tests (backlog#1823)"
|
||||
)]
|
||||
pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
|
||||
Self::new_with_metrics_path_read_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
@@ -420,6 +425,7 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
|
||||
pub fn new_with_metrics_path(
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
e: Erasure,
|
||||
@@ -438,6 +444,7 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
|
||||
pub fn new_with_metrics_path_and_read_costs(
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
e: Erasure,
|
||||
@@ -514,6 +521,7 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
|
||||
fn new_with_read_timeout(
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
e: Erasure,
|
||||
@@ -596,6 +604,7 @@ where
|
||||
verify_reconstruction,
|
||||
locality_preference_enabled: get_shard_locality_preference_enabled(),
|
||||
buffers: ShardBufferPool::new(e.data_shards + e.parity_shards),
|
||||
stripe_state: None,
|
||||
engaged,
|
||||
deferred_handles: Vec::new(),
|
||||
stripe_index: 0,
|
||||
@@ -700,6 +709,12 @@ where
|
||||
{
|
||||
#[hotpath::measure(impl_type = "ParallelReader")]
|
||||
pub async fn read(&mut self) -> StripeReadOutput {
|
||||
let mut state = StripeReadState::with_slot_count(self.readers.len(), self.data_shards);
|
||||
self.read_into_state(&mut state).await;
|
||||
state.into_parts()
|
||||
}
|
||||
|
||||
async fn read_into_state(&mut self, state: &mut StripeReadState) {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
// mutually aligned. The adaptive data-first path below only reads
|
||||
@@ -709,12 +724,14 @@ where
|
||||
// than the data shards, producing "inconsistent read source shards" and
|
||||
// truncating large-object GETs under concurrency (backlog#832).
|
||||
if self.verify_reconstruction {
|
||||
return self.read_lockstep().await;
|
||||
self.read_lockstep(state).await;
|
||||
return;
|
||||
}
|
||||
// if self.readers.len() != self.total_shards {
|
||||
// return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
|
||||
// }
|
||||
let num_readers = self.readers.len();
|
||||
state.reset(num_readers, self.data_shards);
|
||||
|
||||
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
|
||||
self.shard_file_size - self.offset
|
||||
@@ -723,7 +740,7 @@ where
|
||||
};
|
||||
|
||||
if shard_size == 0 {
|
||||
return (smallvec![None; num_readers], smallvec![None; num_readers]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Advance to the next stripe so the following read() computes the correct
|
||||
@@ -734,8 +751,7 @@ where
|
||||
// is only read above to derive `shard_size`, so advancing here is safe.
|
||||
self.offset += shard_size;
|
||||
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
let (shards, errs) = state.parts_mut();
|
||||
let read_costs = self.read_costs.as_slice();
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let low_cost_available = self
|
||||
@@ -882,8 +898,8 @@ where
|
||||
}
|
||||
|
||||
let result_is_err = record_shard_read_result(
|
||||
&mut shards,
|
||||
&mut errs,
|
||||
shards,
|
||||
errs,
|
||||
&mut retire_readers,
|
||||
&mut success,
|
||||
&mut successful_costs,
|
||||
@@ -944,8 +960,8 @@ where
|
||||
active_readers[i] = false;
|
||||
completed += 1;
|
||||
if record_shard_read_result(
|
||||
&mut shards,
|
||||
&mut errs,
|
||||
shards,
|
||||
errs,
|
||||
&mut retire_readers,
|
||||
&mut success,
|
||||
&mut successful_costs,
|
||||
@@ -957,7 +973,7 @@ where
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
retire_abandoned_readers(&mut errs, &mut retire_readers, &active_readers);
|
||||
retire_abandoned_readers(errs, &mut retire_readers, &active_readers);
|
||||
}
|
||||
|
||||
if let Some(path) = self.metrics_path {
|
||||
@@ -1001,8 +1017,6 @@ where
|
||||
for i in retire_readers {
|
||||
self.readers[i] = None;
|
||||
}
|
||||
|
||||
(shards, errs)
|
||||
}
|
||||
|
||||
/// Lockstep stripe read for the reconstruction-verifying GET path.
|
||||
@@ -1030,18 +1044,18 @@ where
|
||||
/// stripe would reintroduce the desync. A parity reader that cannot be
|
||||
/// realigned (no pending deferred handle) is likewise retired instead of
|
||||
/// being read out of position.
|
||||
async fn read_lockstep(&mut self) -> StripeReadOutput {
|
||||
async fn read_lockstep(&mut self, state: &mut StripeReadState) {
|
||||
let num_readers = self.readers.len();
|
||||
state.reset(num_readers, self.data_shards);
|
||||
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
|
||||
self.shard_file_size - self.offset
|
||||
} else {
|
||||
self.shard_size
|
||||
};
|
||||
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
let (shards, errs) = state.parts_mut();
|
||||
if shard_size == 0 {
|
||||
return (shards, errs);
|
||||
return;
|
||||
}
|
||||
|
||||
// Advance to the next stripe (see the matching note in `read`); the
|
||||
@@ -1279,8 +1293,6 @@ where
|
||||
for i in retire_readers {
|
||||
self.readers[i] = None;
|
||||
}
|
||||
|
||||
(shards, errs)
|
||||
}
|
||||
|
||||
/// Attempt to bring an as-yet-unread parity reader into the lockstep read
|
||||
@@ -1326,10 +1338,6 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn can_decode(&self, shards: &[Option<Vec<u8>>]) -> bool {
|
||||
shards.iter().filter(|s| s.is_some()).count() >= self.data_shards
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -1337,10 +1345,20 @@ impl<R> ShardStripeSource for ParallelReader<R>
|
||||
where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
async fn read_next_stripe(&mut self) -> StripeReadState {
|
||||
let read_quorum = self.data_shards;
|
||||
let (shards, errors) = ParallelReader::read(self).await;
|
||||
StripeReadState::from_parts_with_read_costs(shards, errors, &self.read_costs, read_quorum)
|
||||
async fn read_next_stripe(&mut self) -> Box<StripeReadState> {
|
||||
let mut state = self
|
||||
.stripe_state
|
||||
.take()
|
||||
.unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
|
||||
self.read_into_state(&mut state).await;
|
||||
state
|
||||
}
|
||||
|
||||
fn recycle_stripe(&mut self, mut state: Box<StripeReadState>) {
|
||||
self.recycle_shards(state.shards_mut());
|
||||
state.reset(0, self.data_shards);
|
||||
debug_assert!(self.stripe_state.is_none(), "a stripe cannot be recycled twice");
|
||||
self.stripe_state = Some(state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1525,6 +1543,7 @@ impl Erasure {
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) async fn decode_with_read_costs<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
@@ -1595,9 +1614,9 @@ impl Erasure {
|
||||
*ret_err = Some(err.into());
|
||||
}
|
||||
|
||||
// Equivalent to `ParallelReader::can_decode`; inlined so this helper does
|
||||
// not need to borrow the reader, leaving the reader free for the
|
||||
// concurrent next-stripe read under prefetch.
|
||||
// Shard-availability check, written out here rather than called on the
|
||||
// reader so this helper does not need to borrow it, leaving the reader
|
||||
// free for the concurrent next-stripe read under prefetch.
|
||||
let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
|
||||
if available_shards < self.data_shards {
|
||||
let reason = GetObjectFailureReason::ReadQuorum;
|
||||
@@ -1972,13 +1991,18 @@ mod tests {
|
||||
type BoxedShardReader = crate::io_support::bitrot::ShardReader;
|
||||
|
||||
#[test]
|
||||
fn shard_scratch_stays_inline_through_the_common_limit_and_spills_safely() {
|
||||
let inline: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS];
|
||||
assert!(!inline.spilled(), "the common shard-count boundary must not allocate");
|
||||
|
||||
let spilled: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS + 1];
|
||||
assert!(spilled.spilled(), "larger supported shard counts must fall back to the heap");
|
||||
assert_eq!(spilled.len(), INLINE_SHARD_SLOTS + 1);
|
||||
fn parallel_reader_keeps_stripe_scratch_out_of_line() {
|
||||
eprintln!(
|
||||
"parallel_reader={} stripe_state={} cached_state={}",
|
||||
std::mem::size_of::<ParallelReader<Cursor<Vec<u8>>>>(),
|
||||
std::mem::size_of::<StripeReadState>(),
|
||||
std::mem::size_of::<Option<Box<StripeReadState>>>()
|
||||
);
|
||||
assert_eq!(
|
||||
std::mem::size_of::<Option<Box<StripeReadState>>>(),
|
||||
std::mem::size_of::<usize>(),
|
||||
"the request-scoped cache must remain pointer-sized",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1997,6 +2021,62 @@ mod tests {
|
||||
assert_eq!(errors.len(), TOTAL_SHARDS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codec_reader_reuses_inline_and_spilled_stripe_scratch_between_reads() {
|
||||
for total_shards in [INLINE_SHARD_SLOTS, INLINE_SHARD_SLOTS + 1] {
|
||||
let data_shards = total_shards - 1;
|
||||
let readers = std::iter::repeat_with(|| None).take(total_shards).collect();
|
||||
let erasure = Erasure::new(data_shards, 1, data_shards * 2);
|
||||
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, data_shards * 2);
|
||||
|
||||
let first = ShardStripeSource::read_next_stripe(&mut reader).await;
|
||||
let first_state = (&*first) as *const StripeReadState;
|
||||
let first_storage = first.scratch_storage();
|
||||
assert_eq!(first_storage.2, total_shards > INLINE_SHARD_SLOTS);
|
||||
assert_eq!(first_storage.3, total_shards > INLINE_SHARD_SLOTS);
|
||||
ShardStripeSource::recycle_stripe(&mut reader, first);
|
||||
|
||||
let second = ShardStripeSource::read_next_stripe(&mut reader).await;
|
||||
let second_storage = second.scratch_storage();
|
||||
|
||||
assert_eq!(
|
||||
(&*second) as *const StripeReadState,
|
||||
first_state,
|
||||
"the request-scoped state must be reused"
|
||||
);
|
||||
assert_eq!(second_storage.0, first_storage.0, "shard slots must reuse their allocation");
|
||||
assert_eq!(second_storage.1, first_storage.1, "error slots must reuse their allocation");
|
||||
assert_eq!(second.into_parts().0.len(), total_shards);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codec_reader_returns_shard_allocations_to_the_request_pool() {
|
||||
const SHARD_SIZE: usize = 16;
|
||||
let hash_algo = HashAlgorithm::None;
|
||||
let readers = vec![Some(create_reader(SHARD_SIZE, 2, 0x5a, &hash_algo, false).await)];
|
||||
let erasure = Erasure::new(1, 0, SHARD_SIZE);
|
||||
let mut reader = ParallelReader::new(readers, erasure, 0, SHARD_SIZE * 2);
|
||||
|
||||
let first = ShardStripeSource::read_next_stripe(&mut reader).await;
|
||||
let first_allocation = first
|
||||
.shard_allocation(0)
|
||||
.expect("the first stripe should own its shard allocation");
|
||||
ShardStripeSource::recycle_stripe(&mut reader, first);
|
||||
assert_eq!(
|
||||
reader.buffers.stored_allocation(0),
|
||||
Some(first_allocation),
|
||||
"recycling a stripe must return its shard allocation to the request pool"
|
||||
);
|
||||
|
||||
let second = ShardStripeSource::read_next_stripe(&mut reader).await;
|
||||
assert_eq!(
|
||||
second.shard_allocation(0),
|
||||
Some(first_allocation),
|
||||
"the next stripe must reuse the pooled shard allocation"
|
||||
);
|
||||
}
|
||||
|
||||
/// Counts the raw bytes pulled from a shard stream, to prove which shards
|
||||
/// a decode path actually touches (backlog#923 call-count evidence).
|
||||
struct CountingShardReader {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user