Compare commits

..

7 Commits

99 changed files with 3556 additions and 9956 deletions
+1 -7
View File
@@ -252,16 +252,10 @@ test-group = 'ecstore-serial-flaky'
# cluster, so it keeps the lane's parallel-safe / no-external-dependency # cluster, so it keeps the lane's parallel-safe / no-external-dependency
# properties. The RustFS warm backend has no loopback guard (that guard is # properties. The RustFS warm backend has no loopback guard (that guard is
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target. # replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
#
# Disk compression (backlog#1848): the `compression` module joins the smoke
# lane so the multipart disk-compression roundtrips (restored after
# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate.
# Single-node servers on random ports with isolated temp dirs — meets the
# admission criteria unchanged.
[profile.e2e-smoke] [profile.e2e-smoke]
default-filter = """ default-filter = """
package(e2e_test) & ( package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/) test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/) | test(/^reliant::lifecycle::/)
| test(/^reliant::tiering::/) | test(/^reliant::tiering::/)
+1 -25
View File
@@ -94,7 +94,6 @@ jobs:
short_sha: ${{ steps.check.outputs.short_sha }} short_sha: ${{ steps.check.outputs.short_sha }}
is_prerelease: ${{ steps.check.outputs.is_prerelease }} is_prerelease: ${{ steps.check.outputs.is_prerelease }}
create_latest: ${{ steps.check.outputs.create_latest }} create_latest: ${{ steps.check.outputs.create_latest }}
source_ref: ${{ steps.check.outputs.source_ref }}
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -119,7 +118,6 @@ jobs:
short_sha="" short_sha=""
is_prerelease=false is_prerelease=false
create_latest=false create_latest=false
source_ref="$GITHUB_SHA"
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
# Triggered by build workflow completion # Triggered by build workflow completion
@@ -139,7 +137,6 @@ jobs:
# Extract version info from commit message or use commit SHA # Extract version info from commit message or use commit SHA
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml) # Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
short_sha=$(git rev-parse --short "$HEAD_SHA") short_sha=$(git rev-parse --short "$HEAD_SHA")
source_ref="$HEAD_SHA"
# Determine build type based on triggering workflow event and ref # Determine build type based on triggering workflow event and ref
triggering_event="$TRIGGERING_EVENT" triggering_event="$TRIGGERING_EVENT"
@@ -264,23 +261,6 @@ jobs:
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported" echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
;; ;;
esac esac
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
tag_ref="refs/tags/$input_version"
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
if [[ "$input_version" == v* ]]; then
tag_ref="refs/tags/${input_version#v}"
else
tag_ref="refs/tags/v$input_version"
fi
fi
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
echo "❌ Release tag not found for Docker build: $input_version"
exit 1
fi
source_ref="$tag_ref"
fi
fi fi
{ {
@@ -291,7 +271,6 @@ jobs:
echo "short_sha=$short_sha" echo "short_sha=$short_sha"
echo "is_prerelease=$is_prerelease" echo "is_prerelease=$is_prerelease"
echo "create_latest=$create_latest" echo "create_latest=$create_latest"
echo "source_ref=$source_ref"
} >> "$GITHUB_OUTPUT" } >> "$GITHUB_OUTPUT"
echo "🐳 Docker Build Summary:" echo "🐳 Docker Build Summary:"
@@ -302,7 +281,6 @@ jobs:
echo " - Short SHA: $short_sha" echo " - Short SHA: $short_sha"
echo " - Is prerelease: $is_prerelease" echo " - Is prerelease: $is_prerelease"
echo " - Create latest: $create_latest" echo " - Create latest: $create_latest"
echo " - Source ref: $source_ref"
# Build multi-arch Docker images # Build multi-arch Docker images
# Strategy: Build images using pre-built binaries from dl.rustfs.com # Strategy: Build images using pre-built binaries from dl.rustfs.com
@@ -330,7 +308,6 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with: with:
persist-credentials: false persist-credentials: false
ref: ${{ needs.build-check.outputs.source_ref }}
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
@@ -420,8 +397,7 @@ jobs:
LABELS="org.opencontainers.image.title=RustFS" LABELS="org.opencontainers.image.title=RustFS"
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system" LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
LABELS="$LABELS,org.opencontainers.image.version=$VERSION" LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
SOURCE_REVISION="$(git rev-parse HEAD)" LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE" LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
Generated
+47 -47
View File
@@ -3761,7 +3761,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]] [[package]]
name = "e2e_test" name = "e2e_test"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"astral-tokio-tar", "astral-tokio-tar",
@@ -9090,7 +9090,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs" name = "rustfs"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"anyhow", "anyhow",
@@ -9227,7 +9227,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-audit" name = "rustfs-audit"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"const-str", "const-str",
@@ -9250,7 +9250,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-checksums" name = "rustfs-checksums"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"base64-simd", "base64-simd",
"bytes", "bytes",
@@ -9266,7 +9266,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-common" name = "rustfs-common"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"chrono", "chrono",
"hotpath", "hotpath",
@@ -9284,7 +9284,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-concurrency" name = "rustfs-concurrency"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"insta", "insta",
@@ -9297,7 +9297,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-config" name = "rustfs-config"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"const-str", "const-str",
"hotpath", "hotpath",
@@ -9307,7 +9307,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-credentials" name = "rustfs-credentials"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"base64-simd", "base64-simd",
"hmac 0.13.0", "hmac 0.13.0",
@@ -9321,7 +9321,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-crypto" name = "rustfs-crypto"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"argon2", "argon2",
@@ -9342,7 +9342,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-data-usage" name = "rustfs-data-usage"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"rmp-serde", "rmp-serde",
@@ -9352,7 +9352,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-ecstore" name = "rustfs-ecstore"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-channel", "async-channel",
@@ -9491,7 +9491,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-extension-schema" name = "rustfs-extension-schema"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"serde", "serde",
@@ -9501,7 +9501,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-filemeta" name = "rustfs-filemeta"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"byteorder", "byteorder",
@@ -9528,7 +9528,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-heal" name = "rustfs-heal"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.23.1", "base64 0.23.1",
@@ -9559,7 +9559,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-iam" name = "rustfs-iam"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -9600,7 +9600,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-io-core" name = "rustfs-io-core"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"bytes", "bytes",
"hotpath", "hotpath",
@@ -9613,7 +9613,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-io-metrics" name = "rustfs-io-metrics"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"criterion", "criterion",
"hotpath", "hotpath",
@@ -9677,7 +9677,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-keystone" name = "rustfs-keystone"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures", "futures",
@@ -9704,7 +9704,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-kms" name = "rustfs-kms"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"anyhow", "anyhow",
@@ -9753,7 +9753,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-lifecycle" name = "rustfs-lifecycle"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"hotpath", "hotpath",
@@ -9776,7 +9776,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-lock" name = "rustfs-lock"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"compact_str", "compact_str",
@@ -9799,7 +9799,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-log-analyzer" name = "rustfs-log-analyzer"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"chrono", "chrono",
"flate2", "flate2",
@@ -9818,7 +9818,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-madmin" name = "rustfs-madmin"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"humantime", "humantime",
@@ -9833,7 +9833,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-notify" name = "rustfs-notify"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -9868,7 +9868,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-object-capacity" name = "rustfs-object-capacity"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"criterion", "criterion",
"futures", "futures",
@@ -9888,7 +9888,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-object-data-cache" name = "rustfs-object-data-cache"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"bytes", "bytes",
"criterion", "criterion",
@@ -9905,7 +9905,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-obs" name = "rustfs-obs"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"chrono", "chrono",
"crossbeam-channel", "crossbeam-channel",
@@ -9960,7 +9960,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-policy" name = "rustfs-policy"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64-simd", "base64-simd",
@@ -9991,7 +9991,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-protocols" name = "rustfs-protocols"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"astral-tokio-tar", "astral-tokio-tar",
"async-compression", "async-compression",
@@ -10053,7 +10053,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-protos" name = "rustfs-protos"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"flatbuffers", "flatbuffers",
"hotpath", "hotpath",
@@ -10077,7 +10077,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-replication" name = "rustfs-replication"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"bytes", "bytes",
@@ -10095,7 +10095,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-rio" name = "rustfs-rio"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"arc-swap", "arc-swap",
@@ -10133,7 +10133,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-rio-v2" name = "rustfs-rio-v2"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"bytes", "bytes",
@@ -10156,7 +10156,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-s3-ops" name = "rustfs-s3-ops"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"rustfs-s3-types", "rustfs-s3-types",
@@ -10164,7 +10164,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-s3-types" name = "rustfs-s3-types"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"serde", "serde",
@@ -10173,7 +10173,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-s3select-api" name = "rustfs-s3select-api"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"bytes", "bytes",
@@ -10203,7 +10203,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-s3select-query" name = "rustfs-s3select-query"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-recursion", "async-recursion",
"async-trait", "async-trait",
@@ -10222,7 +10222,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-scanner" name = "rustfs-scanner"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"bytes", "bytes",
@@ -10262,7 +10262,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-security-governance" name = "rustfs-security-governance"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"thiserror 2.0.20", "thiserror 2.0.20",
@@ -10270,7 +10270,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-signer" name = "rustfs-signer"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"base64-simd", "base64-simd",
"bytes", "bytes",
@@ -10288,7 +10288,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-storage-api" name = "rustfs-storage-api"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"hotpath", "hotpath",
@@ -10303,7 +10303,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-targets" name = "rustfs-targets"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-nats", "async-nats",
@@ -10357,7 +10357,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-test-utils" name = "rustfs-test-utils"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"rustfs-data-usage", "rustfs-data-usage",
@@ -10373,7 +10373,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-tls-runtime" name = "rustfs-tls-runtime"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"hotpath", "hotpath",
@@ -10394,7 +10394,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-trusted-proxies" name = "rustfs-trusted-proxies"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"axum", "axum",
@@ -10431,7 +10431,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-utils" name = "rustfs-utils"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"base64-simd", "base64-simd",
"blake2", "blake2",
@@ -10473,7 +10473,7 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-zip" name = "rustfs-zip"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
dependencies = [ dependencies = [
"astral-tokio-tar", "astral-tokio-tar",
"async-compression", "async-compression",
+47 -47
View File
@@ -69,7 +69,7 @@ edition = "2024"
license = "Apache-2.0" license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs" repository = "https://github.com/rustfs/rustfs"
rust-version = "1.97.1" rust-version = "1.97.1"
version = "1.0.0-rc.2" version = "1.0.0-rc.1"
homepage = "https://rustfs.com" homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"] keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -86,52 +86,52 @@ redundant_clone = "warn"
[workspace.dependencies] [workspace.dependencies]
# RustFS Internal Crates # RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" } rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" } rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" } rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" } rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" } rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" } rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" } rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" } rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" } rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" } rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" } rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" } rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" } rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" } rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" } rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" } rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" } rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" } rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" } rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" } rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" } rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" } rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false } rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" } rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" } rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" } rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" } rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" } rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" } rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" } rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" } rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" } rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" } rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" } rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" } rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" } rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" } rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" } rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" } rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" } rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" } rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" } rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" } rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" } rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" } rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" } rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
# Async Runtime and Networking # Async Runtime and Networking
async-channel = "2.5.0" async-channel = "2.5.0"
+1 -1
View File
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# Using specific version # Using specific version
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
``` ```
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
+1 -1
View File
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# 使用指定版本运行 # 使用指定版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
``` ```
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录: 如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
+1 -4
View File
@@ -67,10 +67,7 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
} }
pub(crate) fn capture_command_logs( fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
command: &mut Command,
log_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let Some(log_path) = log_path else { let Some(log_path) = log_path else {
return Ok(()); return Ok(());
}; };
+3 -663
View File
@@ -2,7 +2,6 @@
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use serial_test::serial; use serial_test::serial;
use std::fs; use std::fs;
use std::path::PathBuf; use std::path::PathBuf;
@@ -26,15 +25,6 @@ fn generate_compressible_data(size: usize) -> Vec<u8> {
data data
} }
/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part
/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload
/// Pending truncation (rustfs/rustfs#5957).
fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
(0..size)
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
.collect()
}
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> { fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
let bucket_path = PathBuf::from(temp_dir).join(bucket); let bucket_path = PathBuf::from(temp_dir).join(bucket);
let mut part_files = Vec::new(); let mut part_files = Vec::new();
@@ -65,14 +55,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
env.cleanup_existing_processes().await?; env.cleanup_existing_processes().await?;
let binary_path = rustfs_binary_path(); let binary_path = rustfs_binary_path();
// Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR let process = Command::new(&binary_path)
// capture (survives the temp-dir cleanup on Drop and is uploaded as a CI
// artifact); without the env var the child inherits stdio as before.
let mut command = Command::new(&binary_path);
command
.env("RUSTFS_CONSOLE_ENABLE", "false") .env("RUSTFS_CONSOLE_ENABLE", "false")
.env("RUSTFS_COMPRESSION_ENABLED", "true") .env("RUSTFS_COMPRESSION_ENABLED", "true")
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
.args([ .args([
"--address", "--address",
&env.address, &env.address,
@@ -81,9 +66,8 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
"--secret-key", "--secret-key",
&env.secret_key, &env.secret_key,
&env.temp_dir, &env.temp_dir,
]); ])
crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?; .spawn()?;
let process = command.spawn()?;
env.process = Some(process); env.process = Some(process);
@@ -170,647 +154,3 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
env.stop_server(); env.stop_server();
Ok(()) Ok(())
} }
const MULTIPART_COMPRESSION_BUCKET: &str = "compression-multipart-bucket";
const MPU_PART1_SIZE: usize = 5 * 1024 * 1024;
const MPU_PART2_SIZE: usize = 1024 * 1024;
async fn multipart_upload(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
parts: &[&[u8]],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
let mut completed_parts = Vec::with_capacity(parts.len());
for (i, part) in parts.iter().enumerate() {
let part_number = (i + 1) as i32;
let upload = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(&upload_id)
.part_number(part_number)
.body(ByteStream::from(part.to_vec()))
.send()
.await?;
completed_parts.push(
CompletedPart::builder()
.part_number(part_number)
.e_tag(upload.e_tag().unwrap_or_default())
.build(),
);
}
client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(&upload_id)
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
.send()
.await?;
Ok(())
}
async fn fetch_range(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
range: &str,
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
let response = client.get_object().bucket(bucket).key(key).range(range).send().await?;
Ok(response.body.collect().await?.into_bytes().to_vec())
}
/// Multipart disk compression roundtrip: parts are written as independent
/// compressed streams and every GET shape must reassemble the original bytes
/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression
/// entirely).
#[tokio::test]
#[serial]
async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting multipart compression roundtrip test");
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_with_compression(&mut env).await?;
let client = env.create_s3_client();
env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
let object_key = "multipart-compressible.txt";
let part1 = generate_compressible_data(MPU_PART1_SIZE);
let part2 = generate_compressible_data(MPU_PART2_SIZE);
let mut original_data = part1.clone();
original_data.extend_from_slice(&part2);
let total_size = original_data.len();
multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?;
let head_response = client
.head_object()
.bucket(MULTIPART_COMPRESSION_BUCKET)
.key(object_key)
.send()
.await?;
assert_eq!(
head_response.content_length().unwrap_or(0) as usize,
total_size,
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
);
info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)");
// Full GET must reassemble both independently compressed parts.
let get_response = client
.get_object()
.bucket(MULTIPART_COMPRESSION_BUCKET)
.key(object_key)
.send()
.await?;
let downloaded = get_response.body.collect().await?.into_bytes();
assert_eq!(downloaded.len(), total_size);
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
// Range fully inside part 1.
let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?;
assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch");
// Range crossing the part boundary.
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
let range_crossing = fetch_range(
&client,
MULTIPART_COMPRESSION_BUCKET,
object_key,
&format!("bytes={boundary_start}-{boundary_end}"),
)
.await?;
assert_eq!(
&range_crossing[..],
&original_data[boundary_start..boundary_end + 1],
"boundary-crossing range mismatch"
);
// Range fully inside part 2.
let part2_start = MPU_PART1_SIZE + 4096;
let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1;
let range_inside_part2 = fetch_range(
&client,
MULTIPART_COMPRESSION_BUCKET,
object_key,
&format!("bytes={part2_start}-{part2_end}"),
)
.await?;
assert_eq!(
&range_inside_part2[..],
&original_data[part2_start..part2_end + 1],
"part-2 range mismatch"
);
// Suffix range (last 128 KiB, entirely in part 2).
let suffix_len = 128 * 1024;
let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
// partNumber GETs must return each original part.
for (part_number, expected) in [(1, &part1), (2, &part2)] {
let response = client
.get_object()
.bucket(MULTIPART_COMPRESSION_BUCKET)
.key(object_key)
.part_number(part_number)
.send()
.await?;
let body = response.body.collect().await?.into_bytes();
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
}
info!("Multipart compression roundtrip test passed");
env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
env.stop_server();
Ok(())
}
const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket";
/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the
/// disk-compression path runs and each part is stored as many compressed blocks — the shape that
/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return
/// the exact original bytes, and the stored size must show the data really was compressed.
#[tokio::test]
#[serial]
async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting multipart high-ratio binary compression roundtrip test");
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_with_compression(&mut env).await?;
let client = env.create_s3_client();
env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
let object_key = "multipart-high-ratio.txt";
let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7);
let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61);
let mut original_data = part1.clone();
original_data.extend_from_slice(&part2);
let total_size = original_data.len();
multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?;
let head_response = client
.head_object()
.bucket(MPU_HIGH_RATIO_BUCKET)
.key(object_key)
.send()
.await?;
assert_eq!(
head_response.content_length().unwrap_or(0) as usize,
total_size,
"Content-Length should be the logical object size"
);
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
// margin still proves the parts were stored compressed rather than raw or double-encoded.
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size as u64) / 2,
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
);
info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)");
info!("step: full GET");
let get_response = client
.get_object()
.bucket(MPU_HIGH_RATIO_BUCKET)
.key(object_key)
.send()
.await?;
let downloaded = get_response.body.collect().await?.into_bytes();
assert_eq!(downloaded.len(), total_size);
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
// Range crossing the part boundary.
info!("step: boundary range GET");
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
let range_crossing = fetch_range(
&client,
MPU_HIGH_RATIO_BUCKET,
object_key,
&format!("bytes={boundary_start}-{boundary_end}"),
)
.await?;
assert_eq!(
&range_crossing[..],
&original_data[boundary_start..boundary_end + 1],
"boundary-crossing range mismatch"
);
// partNumber GET for the trailing part.
info!("step: partNumber GET");
let part2_response = client
.get_object()
.bucket(MPU_HIGH_RATIO_BUCKET)
.key(object_key)
.part_number(2)
.send()
.await?;
let part2_body = part2_response.body.collect().await?.into_bytes();
assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch");
info!("Multipart high-ratio binary compression roundtrip test passed");
env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
env.stop_server();
Ok(())
}
const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket";
const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024;
const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024;
/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied
/// range must be decompressed on read and re-compressed into the destination part, so the final
/// object has to match "source prefix + uploaded tail" byte for byte.
#[tokio::test]
#[serial]
async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting multipart upload-part-copy compression roundtrip test");
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_with_compression(&mut env).await?;
let client = env.create_s3_client();
env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
// Source object: a plain PUT that goes through the single-stream compression path.
let source_key = "copy-source.txt";
let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE);
client
.put_object()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(source_key)
.body(ByteStream::from(source_data.clone()))
.send()
.await?;
// Destination object: part 1 copied from the source, part 2 uploaded directly.
let target_key = "copy-target.txt";
let part2 = generate_compressible_data(MPU_PART2_SIZE);
let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec();
expected_data.extend_from_slice(&part2);
let total_size = expected_data.len();
let create = client
.create_multipart_upload()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.send()
.await?;
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
let copy_part = client
.upload_part_copy()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.upload_id(&upload_id)
.part_number(1)
.copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}"))
.copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1))
.send()
.await?;
let copy_etag = copy_part
.copy_part_result()
.and_then(|r| r.e_tag())
.ok_or("missing copy part etag")?
.to_string();
let uploaded_part = client
.upload_part()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.upload_id(&upload_id)
.part_number(2)
.body(ByteStream::from(part2.clone()))
.send()
.await?;
client
.complete_multipart_upload()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.upload_id(&upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build())
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(uploaded_part.e_tag().unwrap_or_default())
.build(),
)
.build(),
)
.send()
.await?;
let head_response = client
.head_object()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.send()
.await?;
assert_eq!(
head_response.content_length().unwrap_or(0) as usize,
total_size,
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
);
let get_response = client
.get_object()
.bucket(MPU_COPY_COMPRESSION_BUCKET)
.key(target_key)
.send()
.await?;
let downloaded = get_response.body.collect().await?.into_bytes();
assert_eq!(downloaded.len(), total_size);
assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch");
info!("Multipart upload-part-copy compression roundtrip test passed");
env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
env.stop_server();
Ok(())
}
const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket";
const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024;
/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one
/// compressed part stream, and a suffix range must resolve inside the trailing part.
#[tokio::test]
#[serial]
async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Starting three-part multipart compression partNumber test");
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_with_compression(&mut env).await?;
let client = env.create_s3_client();
env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
let object_key = "multipart-three-parts.txt";
let part1 = generate_compressible_data(MPU_PART1_SIZE);
let part2 = generate_compressible_data(MPU_PART1_SIZE);
let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE);
let mut original_data = part1.clone();
original_data.extend_from_slice(&part2);
original_data.extend_from_slice(&part3);
let total_size = original_data.len();
multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?;
let head_response = client
.head_object()
.bucket(MPU_THREE_PARTS_BUCKET)
.key(object_key)
.send()
.await?;
assert_eq!(
head_response.content_length().unwrap_or(0) as usize,
total_size,
"Content-Length should be the logical object size"
);
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
);
// Every partNumber GET must return exactly the bytes of the corresponding uploaded part.
for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] {
let response = client
.get_object()
.bucket(MPU_THREE_PARTS_BUCKET)
.key(object_key)
.part_number(part_number)
.send()
.await?;
let body = response.body.collect().await?.into_bytes();
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
}
// Suffix range (last 64 KiB) resolves inside the trailing part.
let suffix_len = 64 * 1024;
let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
info!("Three-part multipart compression partNumber test passed");
env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
env.stop_server();
Ok(())
}
const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
async fn start_rustfs_with_compression_and_sse(
env: &mut RustFSTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use base64::Engine;
env.cleanup_existing_processes().await?;
let binary_path = rustfs_binary_path();
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
// Server output goes to a file inside the per-test temp dir so a failing
// run can be diagnosed from the child's logs.
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
let server_log_err = server_log.try_clone()?;
let process = Command::new(&binary_path)
.env("RUSTFS_CONSOLE_ENABLE", "false")
.env("RUSTFS_COMPRESSION_ENABLED", "true")
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
.stdout(std::process::Stdio::from(server_log))
.stderr(std::process::Stdio::from(server_log_err))
.args([
"--address",
&env.address,
"--access-key",
&env.access_key,
"--secret-key",
&env.secret_key,
&env.temp_dir,
])
.spawn()?;
env.process = Some(process);
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
for i in 0..30 {
if TcpStream::connect(&env.address).await.is_ok() {
info!("RustFS server is ready after {} attempts", i + 1);
return Ok(());
}
if i == 29 {
return Err("RustFS server failed to become ready".into());
}
sleep(Duration::from_secs(1)).await;
}
Ok(())
}
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
/// shape must still return the original plaintext bytes. Physical size must shrink because the
/// compression runs before encryption.
#[tokio::test]
#[serial]
async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use aws_sdk_s3::types::ServerSideEncryption;
init_logging();
info!("Starting SSE-S3 multipart compression roundtrip test");
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_with_compression_and_sse(&mut env).await?;
let client = env.create_s3_client();
env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
let object_key = "multipart-sse-compressible.txt";
let part1 = generate_compressible_data(MPU_PART1_SIZE);
let part2 = generate_compressible_data(MPU_PART2_SIZE);
let mut original_data = part1.clone();
original_data.extend_from_slice(&part2);
let total_size = original_data.len();
let create = client
.create_multipart_upload()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.server_side_encryption(ServerSideEncryption::Aes256)
.send()
.await?;
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
let mut completed_parts = Vec::new();
for (i, part) in [&part1, &part2].into_iter().enumerate() {
let part_number = (i + 1) as i32;
let upload = client
.upload_part()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.upload_id(&upload_id)
.part_number(part_number)
.body(ByteStream::from(part.clone()))
.send()
.await?;
completed_parts.push(
CompletedPart::builder()
.part_number(part_number)
.e_tag(upload.e_tag().unwrap_or_default())
.build(),
);
}
client
.complete_multipart_upload()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.upload_id(&upload_id)
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
.send()
.await?;
let head_response = client
.head_object()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.send()
.await?;
assert_eq!(
head_response.content_length().unwrap_or(0) as usize,
total_size,
"Content-Length should be the logical object size"
);
assert_eq!(
head_response.server_side_encryption(),
Some(&ServerSideEncryption::Aes256),
"HEAD must report SSE-S3"
);
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
assert!(
total_physical_size < (total_size / 2) as u64,
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
);
let get_response = client
.get_object()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.send()
.await?;
let downloaded = get_response.body.collect().await?.into_bytes();
assert_eq!(downloaded.len(), total_size);
assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch");
// Range crossing the part boundary must decrypt and decompress across parts.
let boundary_start = MPU_PART1_SIZE - 64 * 1024;
let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1;
let range_crossing = fetch_range(
&client,
MPU_SSE_COMPRESSION_BUCKET,
object_key,
&format!("bytes={boundary_start}-{boundary_end}"),
)
.await?;
assert_eq!(
&range_crossing[..],
&original_data[boundary_start..boundary_end + 1],
"SSE-S3 boundary-crossing range mismatch"
);
// partNumber GET for the trailing part.
let part2_response = client
.get_object()
.bucket(MPU_SSE_COMPRESSION_BUCKET)
.key(object_key)
.part_number(2)
.send()
.await?;
let part2_body = part2_response.body.collect().await?.into_bytes();
assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch");
info!("SSE-S3 multipart compression roundtrip test passed");
env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
env.stop_server();
Ok(())
}
@@ -1828,36 +1828,33 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
Ok(()) Ok(())
} }
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
/// Reverting the multipart compression fix must fail this test.
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult { async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
init_logging(); init_logging();
let collector = OtlpMetricCollector::start().await?; let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?; let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
configure_reader_metric_cluster(&mut cluster, &collector); configure_reader_metric_cluster(&mut cluster, &collector);
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
cluster.start().await?; cluster.start().await?;
let bucket = "inline-multipart-compression-roundtrip"; let bucket = "inline-multipart-compression-fallback";
cluster.create_test_bucket(bucket).await?; cluster.create_test_bucket(bucket).await?;
let client = cluster.create_s3_client(0)?; let client = cluster.create_s3_client(0)?;
let key = "multipart/compressed.txt"; let key = "multipart/compression-disabled.txt";
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?; let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
assert_reader_path( assert_reader_path(
&collector, &collector,
&client, &client,
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED), ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
) )
.await?; .await?;
assert_part_number_reader_path( assert_part_number_reader_path(
&collector, &collector,
&client, &client,
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX), PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX),
) )
.await?; .await?;
@@ -1874,7 +1871,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key); cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
configure_mixed_msgpack_cluster(&mut cluster, &collector)?; configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
cluster.start().await?; cluster.start().await?;
@@ -1894,21 +1890,14 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
ReaderPathExpectation::for_class( ReaderPathExpectation::for_class(
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None), ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
LEGACY_DUPLEX, LEGACY_DUPLEX,
COMPRESSED, MULTIPART,
), ),
) )
.await?; .await?;
assert_part_number_reader_path( assert_part_number_reader_path(
&collector, &collector,
&client, &client,
PartNumberReaderPathExpectation::new( PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
bucket,
multipart_key,
&second_part,
multipart_body.len(),
COMPRESSED,
LEGACY_DUPLEX,
),
) )
.await?; .await?;
assert_msgpack_decode_observed(&collector, &decode_before).await?; assert_msgpack_decode_observed(&collector, &decode_before).await?;
@@ -2364,11 +2353,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
hot_client.create_bucket().bucket(bucket).send().await?; hot_client.create_bucket().bucket(bucket).send().await?;
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?; put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
// `.zip` sits on the disk-compression exclusion list: this test pins let key = "transition/mixed-multipart.bin";
// msgpack compat controls across ILM transition, and a compressed object
// would classify as `compressed` instead of `remote` (and the warm-tier
// read path does not decode compression — tracked separately).
let key = "transition/mixed-multipart.zip";
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?; let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
wait_for_transition(&hot_client, bucket, key, &tier_name).await?; wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
assert!( assert!(
@@ -97,7 +97,7 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestRe
let envs = [ let envs = [
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"), ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"), ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "false"),
("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_ILM_PROCESS_TIME", "1"), ("RUSTFS_ILM_PROCESS_TIME", "1"),
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
@@ -486,6 +486,7 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
/// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop. /// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop.
#[tokio::test] #[tokio::test]
#[serial] #[serial]
#[ignore = "pins rustfs/rustfs#6025: GET on a transitioned managed-SSE object silently returns corrupt bytes (fails with enforcement on AND off, so it is not an authorization regression); un-ignore with the fix"]
async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult { async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult {
init_logging(); init_logging();
+5 -13
View File
@@ -131,8 +131,6 @@ pub mod bucket {
} }
pub mod metadata_sys { pub mod metadata_sys {
#[cfg(feature = "test-util")]
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{ pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
@@ -142,7 +140,7 @@ pub mod bucket {
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
update_quota_if_incarnation, update_under_transaction_lock, update_under_transaction_lock,
}; };
} }
@@ -278,9 +276,7 @@ pub mod cluster {
} }
pub mod compression { pub mod compression {
pub use crate::io_support::compress::{ pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled};
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
};
} }
pub mod config { pub mod config {
@@ -320,7 +316,7 @@ pub mod data_usage {
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer, refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
@@ -407,11 +403,8 @@ pub mod metrics {
} }
pub mod notification { pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{ pub use crate::services::notification_sys::{
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof, NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe, start_remote_version_state_fleet_probe,
}; };
} }
@@ -471,8 +464,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
pub mod test_util { pub mod test_util {
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
} }
} }
@@ -46,13 +46,15 @@ use crate::bucket::lifecycle::transition_transaction::run_transition_transaction
use crate::bucket::object_lock::ObjectLockApi; use crate::bucket::object_lock::ObjectLockApi;
use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning::VersioningApi as _;
use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::client::object_api_utils::new_getobjectreader;
use crate::disk::error::DiskError; use crate::disk::error::DiskError;
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE}; use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::Error; use crate::error::Error;
use crate::error::StorageError; use crate::error::StorageError;
use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down}; use crate::error::{
error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down,
};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
use crate::services::tier::{ use crate::services::tier::{
tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata}, tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata},
warm_backend::WarmBackendGetOpts, warm_backend::WarmBackendGetOpts,
@@ -4398,10 +4400,9 @@ pub async fn get_transitioned_object_reader(
h: &HeaderMap, h: &HeaderMap,
oi: &ObjectInfo, oi: &ObjectInfo,
opts: &ObjectOptions, opts: &ObjectOptions,
resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<GetObjectReader, std::io::Error> { ) -> Result<GetObjectReader, std::io::Error> {
let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); let tier_config_mgr = runtime_sources::tier_config_mgr_handle();
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr, resolver).await get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await
} }
fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> { fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
@@ -4421,10 +4422,6 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
} }
} }
// The resolver joins the tier manager as the second injected port this read
// needs; grouping the request half into a struct would churn every call site of
// a bug fix.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn get_transitioned_object_reader_with_tier_manager( pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
bucket: &str, bucket: &str,
object: &str, object: &str,
@@ -4433,7 +4430,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
oi: &ObjectInfo, oi: &ObjectInfo,
opts: &ObjectOptions, opts: &ObjectOptions,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<GetObjectReader, std::io::Error> { ) -> Result<GetObjectReader, std::io::Error> {
validate_transition_remote_version(oi)?; validate_transition_remote_version(oi)?;
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
@@ -4451,16 +4447,11 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
// The same read plan the local path uses, so the tier fetch is positioned in let ret = new_getobjectreader(rs, oi, opts, h);
// the object's *stored* coordinate system and the stream is handed the same if let Err(err) = ret {
// decrypt/decompress transforms. Reading an encrypted object's ciphertext return Err(error_resp_to_object_err(err, vec![bucket, object]));
// through a plaintext-coordinate range and skipping the transform is how a }
// transitioned SSE object used to come back as silently corrupt bytes of the let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check");
// right length (rustfs/rustfs#6025).
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
.await
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
let mut gopts = WarmBackendGetOpts::default(); let mut gopts = WarmBackendGetOpts::default();
if off >= 0 && length >= 0 { if off >= 0 && length >= 0 {
@@ -4497,10 +4488,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
); );
e e
})?; })?;
let object_reader = plan Ok(attach_tier_operation_lease(get_fn(reader, h.clone()), tgt_client))
.into_object_reader(Box::new(reader), oi)
.map_err(|err| std::io::Error::other(format!("wrapping the tier stream for {bucket}/{object} failed: {err}")))?;
Ok(attach_tier_operation_lease(object_reader, tgt_client))
} }
struct TierOperationLeaseReader { struct TierOperationLeaseReader {
@@ -5788,7 +5776,6 @@ mod tests {
&object_info, &object_info,
&ObjectOptions::default(), &ObjectOptions::default(),
&manager, &manager,
None,
) )
.await .await
.expect("transitioned reader should open"); .expect("transitioned reader should open");
@@ -5853,7 +5840,6 @@ mod tests {
&object_info, &object_info,
&ObjectOptions::default(), &ObjectOptions::default(),
&manager, &manager,
None,
) )
.await .await
{ {
@@ -5894,7 +5880,6 @@ mod tests {
&object_info, &object_info,
&ObjectOptions::default(), &ObjectOptions::default(),
&manager, &manager,
None,
) )
.await .await
{ {
@@ -6132,7 +6117,6 @@ mod tests {
&oi, &oi,
&ObjectOptions::default(), &ObjectOptions::default(),
&manager, &manager,
None,
) )
.await .await
{ {
@@ -6156,7 +6140,6 @@ mod tests {
&oi, &oi,
&ObjectOptions::default(), &ObjectOptions::default(),
&manager, &manager,
None,
) )
.await .await
{ {
+1 -142
View File
@@ -50,72 +50,6 @@ use uuid::Uuid;
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
#[cfg(any(test, feature = "test-util"))]
struct ConfigWriteLockProbeState {
bucket: String,
arrived: tokio::sync::Notify,
}
#[cfg(any(test, feature = "test-util"))]
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))]
pub struct ConfigWriteLockProbe {
state: Arc<ConfigWriteLockProbeState>,
}
#[cfg(any(test, feature = "test-util"))]
impl ConfigWriteLockProbe {
pub fn install(bucket: &str) -> Self {
let state = Arc::new(ConfigWriteLockProbeState {
bucket: bucket.to_string(),
arrived: tokio::sync::Notify::new(),
});
let mut probes = CONFIG_WRITE_LOCK_PROBES
.get_or_init(|| StdMutex::new(Vec::new()))
.lock()
.expect("config write lock probe mutex should not poison");
assert!(
!probes.iter().any(|current| current.bucket == state.bucket),
"config write lock probe must be unique for a bucket"
);
probes.push(Arc::clone(&state));
drop(probes);
Self { state }
}
pub async fn wait_until_attempted(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("bucket config update should attempt the transaction lock");
}
}
#[cfg(any(test, feature = "test-util"))]
impl Drop for ConfigWriteLockProbe {
fn drop(&mut self) {
let mut probes = CONFIG_WRITE_LOCK_PROBES
.get_or_init(|| StdMutex::new(Vec::new()))
.lock()
.expect("config write lock probe mutex should not poison");
probes.retain(|state| !Arc::ptr_eq(state, &self.state));
}
}
#[cfg(any(test, feature = "test-util"))]
fn notify_config_write_lock_attempt(bucket: &str) {
let probe = CONFIG_WRITE_LOCK_PROBES
.get_or_init(|| StdMutex::new(Vec::new()))
.lock()
.expect("config write lock probe mutex should not poison")
.iter()
.find(|probe| probe.bucket == bucket)
.cloned();
if let Some(probe) = probe {
probe.arrived.notify_one();
}
}
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
enum MetadataLoadMode { enum MetadataLoadMode {
Initial, Initial,
@@ -656,31 +590,6 @@ pub async fn update_under_transaction_lock(
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
} }
pub async fn update_quota_if_incarnation(
bucket: &str,
data: Vec<u8>,
expected_incarnation_id: Uuid,
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
) -> Result<OffsetDateTime> {
let sys = get_bucket_metadata_sys()?;
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
sys.clone(),
bucket,
Some(expected_incarnation_id),
))
.await?;
if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
return Err(Error::NamespaceLockQuorumUnavailable {
mode: "quota_capability",
bucket: bucket.to_string(),
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
required: 1,
achieved: 0,
});
}
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
}
pub async fn update_bucket_targets_under_transaction_lock( pub async fn update_bucket_targets_under_transaction_lock(
guard: &BucketMetadataMutationGuard, guard: &BucketMetadataMutationGuard,
bucket: &str, bucket: &str,
@@ -825,26 +734,7 @@ async fn acquire_transaction_lock_with_sys(
let lock = api let lock = api
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
.await?; .await?;
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()); Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
#[cfg(any(test, feature = "test-util"))]
{
tokio::pin!(acquire);
let mut notified = false;
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
std::task::Poll::Pending => {
if !notified {
notify_config_write_lock_attempt(bucket);
notified = true;
}
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
})
.await?;
Ok(guard)
}
#[cfg(not(any(test, feature = "test-util")))]
Ok(acquire.await?)
} }
/// The lock resource name is deliberately still the `bucket-targets` one it /// The lock resource name is deliberately still the `bucket-targets` one it
@@ -999,37 +889,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
} }
} }
/// Re-read the quota configuration and bucket incarnation from the same
/// authoritative metadata blob while the caller holds the bucket metadata
/// transaction read lock.
pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
match bucket_meta_sys
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
.await?
{
BucketMetadataAuthority::Authoritative(metadata)
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
{
Ok((
metadata.quota_config.clone(),
metadata.bucket_incarnation_id,
metadata.quota_config_updated_at,
))
}
BucketMetadataAuthority::Authoritative(_) => {
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
}
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
}
}
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await; let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+2 -38
View File
@@ -52,7 +52,6 @@ impl QuotaChecker {
) -> Result<QuotaCheckResult, QuotaError> { ) -> Result<QuotaCheckResult, QuotaError> {
let start_time = Instant::now(); let start_time = Instant::now();
let quota_config = self.get_quota_config(bucket).await?; let quota_config = self.get_quota_config(bucket).await?;
let uses_durable_reservations = quota_config.uses_durable_reservations();
// If no quota limit is set, allow operation // If no quota limit is set, allow operation
let quota_limit = match quota_config.quota { let quota_limit = match quota_config.quota {
@@ -68,7 +67,6 @@ impl QuotaChecker {
quota_limit: None, quota_limit: None,
operation_size, operation_size,
remaining: None, remaining: None,
uses_durable_reservations,
}); });
} }
Some(q) => q, Some(q) => q,
@@ -76,17 +74,14 @@ impl QuotaChecker {
let current_usage = self.get_real_time_usage(bucket).await?; let current_usage = self.get_real_time_usage(bucket).await?;
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
let expected_usage = match operation { let expected_usage = match operation {
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
current_usage.saturating_add(admission_size)
}
QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size), QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
}; };
let allowed = match operation { let allowed = match operation {
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
quota_config.check_operation_allowed(current_usage, admission_size) quota_config.check_operation_allowed(current_usage, operation_size)
} }
QuotaOperation::DeleteObject => true, QuotaOperation::DeleteObject => true,
}; };
@@ -110,7 +105,6 @@ impl QuotaChecker {
quota_limit: Some(quota_limit), quota_limit: Some(quota_limit),
operation_size, operation_size,
remaining, remaining,
uses_durable_reservations,
}; };
let duration = start_time.elapsed(); let duration = start_time.elapsed();
@@ -164,26 +158,6 @@ impl QuotaChecker {
.await .await
} }
pub async fn set_durable_quota_config_if_incarnation(
&mut self,
bucket: &str,
quota: BucketQuota,
expected_incarnation_id: uuid::Uuid,
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
) -> Result<OffsetDateTime, QuotaError> {
let json_data = serde_json::to_vec(&quota).map_err(|e| QuotaError::InvalidConfig {
reason: format!("Failed to serialize quota config: {}", e),
})?;
let start_time = Instant::now();
let updated_at =
crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
.await
.map_err(QuotaError::StorageError)?;
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(updated_at)
}
async fn set_quota_config_for_incarnation( async fn set_quota_config_for_incarnation(
&mut self, &mut self,
bucket: &str, bucket: &str,
@@ -381,7 +355,6 @@ mod tests {
quota_limit: None, quota_limit: None,
operation_size: 1024, operation_size: 1024,
remaining: None, remaining: None,
uses_durable_reservations: false,
}; };
assert!(result.allowed); assert!(result.allowed);
@@ -405,13 +378,4 @@ mod tests {
let allowed = quota.check_operation_allowed(512, 1024); let allowed = quota.check_operation_allowed(512, 1024);
assert!(!allowed); assert!(!allowed);
} }
#[test]
fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
let durable = BucketQuota::new(Some(5));
assert!(!legacy.check_operation_allowed(4, 2));
assert!(durable.uses_durable_reservations());
}
} }
+8 -134
View File
@@ -13,98 +13,38 @@
// limitations under the License. // limitations under the License.
pub mod checker; pub mod checker;
pub(crate) mod reservation;
use crate::error::Result; use crate::error::Result;
use rustfs_config::{ use rustfs_config::{
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE, QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
QUOTA_NOT_FOUND_ERROR_CODE, QUOTA_NOT_FOUND_ERROR_CODE,
}; };
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use time::OffsetDateTime; use time::OffsetDateTime;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub enum QuotaType { pub enum QuotaType {
/// Hard quota accounting. /// Hard quota: reject immediately when exceeded
#[default] #[default]
#[serde(alias = "HARD", alias = "hard")] #[serde(alias = "HARD", alias = "hard")]
Hard, Hard,
} }
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
/// Bucket quota configuration. quota_type defaults to Hard when omitted. /// Bucket quota configuration. quota_type defaults to Hard when omitted.
#[derive(Debug, Default, Clone, PartialEq)] #[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
pub struct BucketQuota { pub struct BucketQuota {
#[serde(default)]
pub quota: Option<u64>, pub quota: Option<u64>,
/// Defaults to Hard when missing. /// Defaults to Hard when missing.
#[serde(default)]
pub quota_type: QuotaType, pub quota_type: QuotaType,
/// Optional durable reservation protocol. The wire format gives older
/// nodes a zero hard quota so a mixed-version fleet fails closed.
pub reservation_protocol: Option<u32>,
/// Timestamp when this quota configuration was set (for audit purposes) /// Timestamp when this quota configuration was set (for audit purposes)
#[serde(default, with = "time::serde::rfc3339::option")]
pub created_at: Option<OffsetDateTime>, pub created_at: Option<OffsetDateTime>,
/// Accept updated_at for compatibility; not used. /// Accept updated_at for compatibility; not used.
pub updated_at: Option<OffsetDateTime>,
}
#[derive(Deserialize, Serialize)]
struct BucketQuotaWire {
#[serde(default)]
quota: Option<u64>,
#[serde(default)]
quota_type: QuotaType,
#[serde(default, skip_serializing_if = "Option::is_none")]
reservation_protocol: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
reservation_quota: Option<u64>,
#[serde(default, with = "time::serde::rfc3339::option")]
created_at: Option<OffsetDateTime>,
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
updated_at: Option<OffsetDateTime>, pub updated_at: Option<OffsetDateTime>,
}
impl Serialize for BucketQuota {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
let durable = self.uses_durable_reservations();
BucketQuotaWire {
quota: if durable { Some(0) } else { self.quota },
quota_type: self.quota_type.clone(),
reservation_protocol: self.reservation_protocol,
reservation_quota: if durable { self.quota } else { None },
created_at: self.created_at,
updated_at: self.updated_at,
}
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for BucketQuota {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = BucketQuotaWire::deserialize(deserializer)?;
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
Some(
wire.reservation_quota
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
)
} else {
wire.quota
};
Ok(Self {
quota,
quota_type: wire.quota_type,
reservation_protocol: wire.reservation_protocol,
created_at: wire.created_at,
updated_at: wire.updated_at,
})
}
} }
impl BucketQuota { impl BucketQuota {
@@ -123,7 +63,6 @@ impl BucketQuota {
Self { Self {
quota, quota,
quota_type: QuotaType::Hard, quota_type: QuotaType::Hard,
reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
created_at: Some(now), created_at: Some(now),
updated_at: None, updated_at: None,
} }
@@ -133,19 +72,7 @@ impl BucketQuota {
self.quota self.quota
} }
pub fn uses_durable_reservations(&self) -> bool {
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
}
pub fn has_unsupported_reservation_protocol(&self) -> bool {
self.reservation_protocol
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
}
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool { pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
if operation_size == 0 {
return true;
}
if let Some(quota_limit) = self.quota { if let Some(quota_limit) = self.quota {
current_usage.saturating_add(operation_size) <= quota_limit current_usage.saturating_add(operation_size) <= quota_limit
} else { } else {
@@ -167,7 +94,6 @@ pub struct QuotaCheckResult {
pub quota_limit: Option<u64>, pub quota_limit: Option<u64>,
pub operation_size: u64, pub operation_size: u64,
pub remaining: Option<u64>, pub remaining: Option<u64>,
pub uses_durable_reservations: bool,
} }
#[derive(Debug)] #[derive(Debug)]
@@ -284,59 +210,7 @@ mod tests {
let buf = q.marshal_msg().expect("marshal"); let buf = q.marshal_msg().expect("marshal");
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal"); let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
assert_eq!(q.quota, restored.quota); assert_eq!(q.quota, restored.quota);
assert_eq!(restored.quota_type, QuotaType::Hard); assert_eq!(q.quota_type, restored.quota_type);
assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
}
#[test]
fn clearing_quota_keeps_the_legacy_compatible_type() {
let quota = BucketQuota::new(None);
assert_eq!(quota.quota_type, QuotaType::Hard);
assert_eq!(quota.reservation_protocol, None);
assert!(!quota.uses_durable_reservations());
}
#[test]
fn durable_quota_makes_legacy_nodes_fail_closed() {
let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
assert!(quota.uses_durable_reservations());
assert_eq!(quota.quota, Some(2048));
#[derive(Deserialize)]
enum LegacyQuotaType {
Hard,
}
#[derive(Deserialize)]
struct LegacyBucketQuota {
#[allow(dead_code)]
quota: Option<u64>,
#[allow(dead_code)]
quota_type: LegacyQuotaType,
}
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
.expect("legacy readers should ignore the reservation protocol field");
assert_eq!(legacy.quota, Some(0));
assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
}
#[test]
fn unknown_reservation_protocol_does_not_activate_v1() {
let quota: BucketQuota =
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
.expect("future protocol should remain parseable");
assert!(!quota.uses_durable_reservations());
assert!(quota.has_unsupported_reservation_protocol());
}
#[test]
fn reservation_protocol_v1_requires_reservation_quota() {
let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
.expect_err("v1 without its authoritative quota must fail closed");
assert!(err.to_string().contains("reservation_quota is required"));
} }
/// unmarshal accepts format without quota_type /// unmarshal accepts format without quota_type
File diff suppressed because it is too large Load Diff
@@ -233,11 +233,17 @@ pub struct NsScannerCapabilityRequest {
#[async_trait] #[async_trait]
pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>; async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
self.open_read(request).await
}
/// Opens an owned-chunk stream when this transport can retain receive-buffer /// Opens an owned-chunk stream when this transport can retain receive-buffer
/// ownership. `None` preserves the established `open_read` fallback. /// ownership. `None` preserves the established `open_read` fallback.
async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> { async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
Ok(None) Ok(None)
} }
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
self.open_read_chunks(request).await
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>; async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>; async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> { async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
@@ -269,6 +275,15 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
)) ))
} }
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
let url = build_read_file_stream_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
Ok(Box::new(
HttpReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout).await?,
))
}
async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> { async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
let url = build_read_file_stream_url(&request); let url = build_read_file_stream_url(&request);
let mut headers = json_headers(); let mut headers = json_headers();
@@ -278,6 +293,16 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
))) )))
} }
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
let url = build_read_file_stream_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
Ok(Some(Box::new(
HttpChunkReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout)
.await?,
)))
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> { async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?; let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
let nonce = server_epoch.map(|_| Uuid::new_v4()); let nonce = server_epoch.map(|_| Uuid::new_v4());
@@ -248,16 +248,6 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8])
Ok(server_epoch) Ok(server_epoch)
} }
fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> {
let version = result
.get(..4)
.and_then(|value| value.try_into().ok())
.map(u32::from_be_bytes)
.ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?;
let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?;
Ok((version, epoch))
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct PeerLiveEventsBatch { pub struct PeerLiveEventsBatch {
pub events: Vec<u8>, pub events: Vec<u8>,
@@ -1298,16 +1288,6 @@ impl PeerRestClient {
Ok((self.topology_member.clone(), epoch)) Ok((self.topology_member.clone(), epoch))
} }
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
probe.extend_from_slice(Uuid::new_v4().as_bytes());
let result = self
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
.await?;
let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?;
Ok((self.topology_member.clone(), supported_version, epoch))
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result( self.finalize_result(
async { async {
@@ -2758,24 +2738,6 @@ mod tests {
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err()); assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
} }
#[test]
fn cross_pool_fence_capability_decoder_fails_closed() {
let epoch = Uuid::new_v4();
let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes())
.expect("small capability response should encode");
assert_eq!(
decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"),
(1, epoch)
);
for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] {
assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err());
}
assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err());
let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes())
.expect("small capability response should encode");
assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err());
}
struct TierMutationResponseFixture<'a> { struct TierMutationResponseFixture<'a> {
version: u32, version: u32,
phase: TierMutationRpcPhase, phase: TierMutationRpcPhase,
+563 -74
View File
@@ -57,15 +57,17 @@ use serde::{Serialize, de::DeserializeOwned};
use std::{ use std::{
io::Cursor, io::Cursor,
path::PathBuf, path::PathBuf,
pin::Pin,
sync::{ sync::{
Arc, Arc,
atomic::{AtomicU32, Ordering}, atomic::{AtomicU32, Ordering},
}, },
task::{Context, Poll},
time::Duration, time::Duration,
}; };
use tokio::time; use tokio::time;
use tokio::{ use tokio::{
io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf},
net::TcpStream, net::TcpStream,
time::timeout, time::timeout,
}; };
@@ -214,6 +216,231 @@ where
} }
} }
fn is_retryable_remote_body_error(error: &io::Error) -> bool {
if error
.get_ref()
.and_then(|source| source.downcast_ref::<rustfs_rio::BodyStalled>())
.is_some()
{
return true;
}
matches!(
error.kind(),
io::ErrorKind::ConnectionReset
| io::ErrorKind::BrokenPipe
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::UnexpectedEof
)
}
fn resumed_read_request(request: &ReadStreamRequest, emitted: usize) -> io::Result<ReadStreamRequest> {
let offset = request
.offset
.checked_add(emitted)
.ok_or_else(|| io::Error::other("remote read resume offset overflow"))?;
let length = if request.length == 0 {
0
} else {
request
.length
.checked_sub(emitted)
.ok_or_else(|| io::Error::other("remote read resume offset exceeds requested length"))?
};
Ok(ReadStreamRequest {
offset,
length,
..request.clone()
})
}
type ReadResumeFuture = tokio::task::JoinHandle<Result<FileReader>>;
struct RetryingRemoteReader {
reader: Option<FileReader>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
emitted: usize,
retried: bool,
resume: Option<ReadResumeFuture>,
}
impl RetryingRemoteReader {
fn new(reader: FileReader, transport: Arc<dyn InternodeDataTransport>, request: ReadStreamRequest) -> Self {
Self {
reader: Some(reader),
transport,
request,
emitted: 0,
retried: false,
resume: None,
}
}
fn start_resume(&mut self) -> io::Result<()> {
if self.request.length != 0 && self.emitted >= self.request.length {
self.reader = None;
return Ok(());
}
let request = resumed_read_request(&self.request, self.emitted)?;
let transport = Arc::clone(&self.transport);
self.resume = Some(tokio::spawn(async move { transport.open_read_fresh(request).await }));
Ok(())
}
}
impl AsyncRead for RetryingRemoteReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
loop {
if let Some(resume) = self.resume.as_mut() {
match Pin::new(resume).poll(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(Ok(reader))) => {
self.resume = None;
self.reader = Some(reader);
}
Poll::Ready(Ok(Err(error))) => {
self.resume = None;
return Poll::Ready(Err(io::Error::other(error)));
}
Poll::Ready(Err(error)) => {
self.resume = None;
return Poll::Ready(Err(io::Error::other(error)));
}
}
}
let Some(reader) = self.reader.as_mut() else {
return Poll::Ready(Ok(()));
};
let before = buf.filled().len();
match Pin::new(reader).poll_read(cx, buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let produced = buf.filled().len() - before;
self.emitted = match self.emitted.checked_add(produced) {
Some(emitted) => emitted,
None => return Poll::Ready(Err(io::Error::other("remote read emitted byte count overflow"))),
};
return Poll::Ready(Ok(()));
}
Poll::Ready(Err(error)) if !self.retried && is_retryable_remote_body_error(&error) => {
self.retried = true;
if let Err(resume_error) = self.start_resume() {
return Poll::Ready(Err(resume_error));
}
continue;
}
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
}
}
}
}
type ChunkResumeFuture = tokio::task::JoinHandle<Result<Option<rustfs_rio::ChunkReaderBox>>>;
struct RetryingRemoteChunkReader {
reader: Option<rustfs_rio::ChunkReaderBox>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
emitted: usize,
retried: bool,
resume: Option<ChunkResumeFuture>,
}
impl RetryingRemoteChunkReader {
fn new(reader: rustfs_rio::ChunkReaderBox, transport: Arc<dyn InternodeDataTransport>, request: ReadStreamRequest) -> Self {
Self {
reader: Some(reader),
transport,
request,
emitted: 0,
retried: false,
resume: None,
}
}
fn start_resume(&mut self) -> io::Result<()> {
if self.request.length != 0 && self.emitted >= self.request.length {
self.reader = None;
return Ok(());
}
let request = resumed_read_request(&self.request, self.emitted)?;
let transport = Arc::clone(&self.transport);
self.resume = Some(tokio::spawn(async move { transport.open_read_chunks_fresh(request).await }));
Ok(())
}
}
impl AsyncRead for RetryingRemoteChunkReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if buf.remaining() == 0 {
return Poll::Ready(Ok(()));
}
match rustfs_rio::ChunkReader::poll_read_chunk(self.as_mut(), cx, buf.remaining()) {
Poll::Ready(Ok(Some(chunk))) => {
buf.put_slice(&chunk);
Poll::Ready(Ok(()))
}
Poll::Ready(Ok(None)) => Poll::Ready(Ok(())),
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
}
impl rustfs_rio::ChunkReader for RetryingRemoteChunkReader {
fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<Bytes>>> {
loop {
if let Some(resume) = self.resume.as_mut() {
match Pin::new(resume).poll(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(Ok(Some(reader)))) => {
self.resume = None;
self.reader = Some(reader);
}
Poll::Ready(Ok(Ok(None))) => {
self.resume = None;
self.reader = None;
return Poll::Ready(Err(io::Error::other("remote resume transport did not provide a chunk reader")));
}
Poll::Ready(Ok(Err(error))) => {
self.resume = None;
return Poll::Ready(Err(io::Error::other(error)));
}
Poll::Ready(Err(error)) => {
self.resume = None;
return Poll::Ready(Err(io::Error::other(error)));
}
}
}
let Some(reader) = self.reader.as_mut() else {
return Poll::Ready(Ok(None));
};
match rustfs_rio::ChunkReader::poll_read_chunk(Pin::new(reader.as_mut()), cx, max) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(Some(chunk))) => {
self.emitted = match self.emitted.checked_add(chunk.len()) {
Some(emitted) => emitted,
None => return Poll::Ready(Err(io::Error::other("remote read emitted byte count overflow"))),
};
return Poll::Ready(Ok(Some(chunk)));
}
Poll::Ready(Ok(None)) => return Poll::Ready(Ok(None)),
Poll::Ready(Err(error)) if !self.retried && is_retryable_remote_body_error(&error) => {
self.retried = true;
if let Err(resume_error) = self.start_resume() {
return Poll::Ready(Err(resume_error));
}
continue;
}
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
}
}
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct RemoteDisk { pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>, pub id: Mutex<Option<Uuid>>,
@@ -1359,71 +1586,6 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into) file_info.validate_for_metadata_read().map_err(Into::into)
} }
impl RemoteDisk {
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
trace!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_data",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout_for_op(
"rename_data",
|| async {
let file_info = compat_json(fi)?;
let file_info_bin = encode_file_info_msgpack(fi)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
});
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)?;
Ok(rename_data_resp)
},
get_max_timeout_duration(),
)
.await
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl DiskAPI for RemoteDisk { impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
@@ -2349,8 +2511,58 @@ impl DiskAPI for RemoteDisk {
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) trace!(
.await event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_data",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout_for_op(
"rename_data",
|| async {
let file_info = compat_json(&fi)?;
let file_info_bin = encode_file_info_msgpack(&fi)?;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
});
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)?;
Ok(rename_data_resp)
},
get_max_timeout_duration(),
)
.await
} }
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
@@ -2484,7 +2696,7 @@ impl DiskAPI for RemoteDisk {
} }
let disk = self.disk_ref().await; let disk = self.disk_ref().await;
let stall_timeout = get_object_disk_read_timeout(); let stall_timeout = get_object_disk_read_timeout();
self.open_read_with_retry(ReadStreamRequest { let request = ReadStreamRequest {
endpoint: self.endpoint.grid_host(), endpoint: self.endpoint.grid_host(),
disk, disk,
volume: volume.to_string(), volume: volume.to_string(),
@@ -2492,8 +2704,9 @@ impl DiskAPI for RemoteDisk {
offset, offset,
length, length,
stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout), stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout),
}) };
.await let reader = self.open_read_with_retry(request.clone()).await?;
Ok(Box::new(RetryingRemoteReader::new(reader, Arc::clone(&self.data_transport), request)))
} }
async fn read_file_stream_chunks( async fn read_file_stream_chunks(
@@ -2508,7 +2721,7 @@ impl DiskAPI for RemoteDisk {
} }
let disk = self.disk_ref().await; let disk = self.disk_ref().await;
let stall_timeout = get_object_disk_read_timeout(); let stall_timeout = get_object_disk_read_timeout();
self.open_read_chunks_with_retry(ReadStreamRequest { let request = ReadStreamRequest {
endpoint: self.endpoint.grid_host(), endpoint: self.endpoint.grid_host(),
disk, disk,
volume: volume.to_string(), volume: volume.to_string(),
@@ -2516,8 +2729,12 @@ impl DiskAPI for RemoteDisk {
offset, offset,
length, length,
stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout), stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout),
}) };
.await let reader = self.open_read_chunks_with_retry(request.clone()).await?;
Ok(reader.map(|reader| {
Box::new(RetryingRemoteChunkReader::new(reader, Arc::clone(&self.data_transport), request))
as rustfs_rio::ChunkReaderBox
}))
} }
/// Buffered read for remote disks. /// Buffered read for remote disks.
@@ -4138,6 +4355,278 @@ mod tests {
} }
} }
#[derive(Debug, Clone)]
enum ResumeReadStep {
PartialThenReset(Vec<u8>),
Data(Vec<u8>),
}
#[derive(Debug, Default)]
struct ResumeTransport {
read_steps: Mutex<Vec<ResumeReadStep>>,
chunk_steps: Mutex<Vec<ResumeReadStep>>,
read_requests: Mutex<Vec<ReadStreamRequest>>,
chunk_requests: Mutex<Vec<ReadStreamRequest>>,
fresh_read_requests: Mutex<Vec<ReadStreamRequest>>,
fresh_chunk_requests: Mutex<Vec<ReadStreamRequest>>,
}
impl ResumeTransport {
fn with_read_steps(read_steps: Vec<ResumeReadStep>) -> Self {
Self {
read_steps: Mutex::new(read_steps),
..Self::default()
}
}
fn with_chunk_steps(chunk_steps: Vec<ResumeReadStep>) -> Self {
Self {
chunk_steps: Mutex::new(chunk_steps),
..Self::default()
}
}
}
#[derive(Debug)]
struct ChunkPartialThenErrorReader {
data: Option<Bytes>,
error: Option<io::Error>,
}
impl rustfs_rio::ChunkReader for ChunkPartialThenErrorReader {
fn poll_read_chunk(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<Bytes>>> {
if let Some(mut data) = self.data.take() {
let take = data.len().min(max);
let chunk = data.split_to(take);
if !data.is_empty() {
self.data = Some(data);
}
return Poll::Ready(Ok(Some(chunk)));
}
if let Some(error) = self.error.take() {
return Poll::Ready(Err(error));
}
Poll::Ready(Ok(None))
}
}
impl AsyncRead for ChunkPartialThenErrorReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Err(io::Error::other("chunk reader must use chunk handoff")))
}
}
fn resume_step_reader(step: ResumeReadStep) -> FileReader {
match step {
ResumeReadStep::PartialThenReset(data) => Box::new(PartialThenErrorReader {
cursor: Cursor::new(data),
error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")),
}),
ResumeReadStep::Data(data) => Box::new(Cursor::new(data)),
}
}
fn resume_step_chunk_reader(step: ResumeReadStep) -> rustfs_rio::ChunkReaderBox {
match step {
ResumeReadStep::PartialThenReset(data) => Box::new(ChunkPartialThenErrorReader {
data: Some(Bytes::from(data)),
error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")),
}),
ResumeReadStep::Data(data) => Box::new(ChunkPartialThenErrorReader {
data: Some(Bytes::from(data)),
error: None,
}),
}
}
#[async_trait::async_trait]
impl InternodeDataTransport for ResumeTransport {
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader> {
self.read_requests
.lock()
.expect("read request lock should not be poisoned")
.push(request);
let step = self
.read_steps
.lock()
.expect("read steps lock should not be poisoned")
.remove(0);
Ok(resume_step_reader(step))
}
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
self.fresh_read_requests
.lock()
.expect("fresh read request lock should not be poisoned")
.push(request.clone());
self.open_read(request).await
}
async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
self.chunk_requests
.lock()
.expect("chunk request lock should not be poisoned")
.push(request);
let step = self
.chunk_steps
.lock()
.expect("chunk steps lock should not be poisoned")
.remove(0);
Ok(Some(resume_step_chunk_reader(step)))
}
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
self.fresh_chunk_requests
.lock()
.expect("fresh chunk request lock should not be poisoned")
.push(request.clone());
self.open_read_chunks(request).await
}
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
panic!("open_write should not be used in remote read resume tests");
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
panic!("open_walk_dir should not be used in remote read resume tests");
}
fn name(&self) -> &'static str {
"resume-test"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
fn resume_request(length: usize) -> ReadStreamRequest {
ReadStreamRequest {
endpoint: "http://remote".to_string(),
disk: "disk".to_string(),
volume: "volume".to_string(),
path: "path".to_string(),
offset: 7,
length,
stall_timeout: None,
}
}
#[tokio::test]
async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
let request = resume_request(10);
let reader = resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec()));
let mut reader = RetryingRemoteReader::new(reader, transport.clone(), request);
let mut output = Vec::new();
reader
.read_to_end(&mut output)
.await
.expect("one body reset should be resumed");
assert_eq!(output, b"0123456789");
let requests = transport
.read_requests
.lock()
.expect("read request lock should not be poisoned");
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].offset, 11);
assert_eq!(requests[0].length, 6);
assert_eq!(
transport
.fresh_read_requests
.lock()
.expect("fresh read request lock should not be poisoned")
.len(),
1
);
}
#[tokio::test]
async fn remote_chunk_reader_resumes_from_emitted_bytes_without_duplicates() {
let transport = Arc::new(ResumeTransport::with_chunk_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
let request = resume_request(10);
let reader = resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec()));
let mut reader = RetryingRemoteChunkReader::new(reader, transport.clone(), request);
let mut output = Vec::new();
reader
.read_to_end(&mut output)
.await
.expect("chunk body reset should be resumed");
assert_eq!(output, b"0123456789");
let requests = transport
.chunk_requests
.lock()
.expect("chunk request lock should not be poisoned");
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].offset, 11);
assert_eq!(requests[0].length, 6);
assert_eq!(
transport
.fresh_chunk_requests
.lock()
.expect("fresh chunk request lock should not be poisoned")
.len(),
1
);
}
#[tokio::test]
async fn remote_reader_retries_at_most_once_and_preserves_non_retryable_errors() {
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::PartialThenReset(b"456".to_vec())]));
let mut reader = RetryingRemoteReader::new(
resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
transport.clone(),
resume_request(7),
);
let error = reader
.read_to_end(&mut Vec::new())
.await
.expect_err("second reset must not retry");
assert_eq!(error.kind(), std_io::ErrorKind::ConnectionReset);
assert_eq!(
transport
.read_requests
.lock()
.expect("read request lock should not be poisoned")
.len(),
1
);
let transport = Arc::new(ResumeTransport::default());
let reader = PartialThenErrorReader {
cursor: Cursor::new(b"data".to_vec()),
error: Some(io::Error::new(std_io::ErrorKind::PermissionDenied, "permission denied")),
};
let mut reader = RetryingRemoteReader::new(Box::new(reader), transport.clone(), resume_request(4));
let error = reader
.read_to_end(&mut Vec::new())
.await
.expect_err("non-retryable errors must not retry");
assert_eq!(error.kind(), std_io::ErrorKind::PermissionDenied);
assert!(
transport
.read_requests
.lock()
.expect("read request lock should not be poisoned")
.is_empty()
);
}
#[test]
fn resumed_read_request_checks_large_offsets() {
let request = ReadStreamRequest {
offset: usize::MAX - 1,
length: 0,
..resume_request(0)
};
assert!(resumed_read_request(&request, 2).is_err());
let request = resume_request(4);
assert!(resumed_read_request(&request, 5).is_err());
}
fn init_tracing(filter_level: Level) { fn init_tracing(filter_level: Level) {
INIT.call_once(|| { INIT.call_once(|| {
let _ = tracing_subscriber::fmt() let _ = tracing_subscriber::fmt()
@@ -12,16 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//! Per-disk usage snapshots persisted under the metadata bucket.
//!
//! **Nothing calls into this module.** It landed complete with tests in #5307
//! (2026-07-27) and its aggregation entry point,
//! [`crate::data_usage::aggregate_local_snapshots`], has never had a caller in
//! the tree's history. The live data-usage path is
//! `load_data_usage_from_backend` / `store_data_usage_in_backend`. The items
//! below therefore carry individual `dead_code` allows rather than a module
//! blanket, so the gap stays greppable until it is either wired up or removed.
use crate::data_usage::BucketUsageInfo; use crate::data_usage::BucketUsageInfo;
use crate::disk::RUSTFS_META_BUCKET; use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
@@ -36,12 +26,10 @@ pub const DATA_USAGE_DIR: &str = "datausage";
/// Directory used to store incremental scan state files under the metadata bucket. /// Directory used to store incremental scan state files under the metadata bucket.
pub const DATA_USAGE_STATE_DIR: &str = "datausage/state"; pub const DATA_USAGE_STATE_DIR: &str = "datausage/state";
/// Snapshot file format version, allows forward compatibility if the structure evolves. /// Snapshot file format version, allows forward compatibility if the structure evolves.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub const LOCAL_USAGE_SNAPSHOT_VERSION: u32 = 1; pub const LOCAL_USAGE_SNAPSHOT_VERSION: u32 = 1;
/// Additional metadata describing which disk produced the snapshot. /// Additional metadata describing which disk produced the snapshot.
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub struct LocalUsageSnapshotMeta { pub struct LocalUsageSnapshotMeta {
/// Disk UUID stored as a string for simpler serialization. /// Disk UUID stored as a string for simpler serialization.
pub disk_id: String, pub disk_id: String,
@@ -55,7 +43,6 @@ pub struct LocalUsageSnapshotMeta {
/// Usage snapshot produced by a single disk. /// Usage snapshot produced by a single disk.
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub struct LocalUsageSnapshot { pub struct LocalUsageSnapshot {
/// Format version recorded in the snapshot. /// Format version recorded in the snapshot.
pub format_version: u32, pub format_version: u32,
@@ -77,7 +64,6 @@ pub struct LocalUsageSnapshot {
pub objects_total_size: u64, pub objects_total_size: u64,
} }
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
impl LocalUsageSnapshot { impl LocalUsageSnapshot {
/// Create an empty snapshot with the default format version filled in. /// Create an empty snapshot with the default format version filled in.
pub fn new(meta: LocalUsageSnapshotMeta) -> Self { pub fn new(meta: LocalUsageSnapshotMeta) -> Self {
@@ -113,13 +99,11 @@ impl LocalUsageSnapshot {
} }
/// Build the snapshot file name `<disk-id>.json`. /// Build the snapshot file name `<disk-id>.json`.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub fn snapshot_file_name(disk_id: &str) -> String { pub fn snapshot_file_name(disk_id: &str) -> String {
format!("{disk_id}.json") format!("{disk_id}.json")
} }
/// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/<disk-id>.json`. /// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/<disk-id>.json`.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub fn snapshot_object_path(disk_id: &str) -> String { pub fn snapshot_object_path(disk_id: &str) -> String {
format!("{}/{}", DATA_USAGE_DIR, snapshot_file_name(disk_id)) format!("{}/{}", DATA_USAGE_DIR, snapshot_file_name(disk_id))
} }
@@ -135,13 +119,11 @@ pub fn data_usage_state_dir(root: &Path) -> PathBuf {
} }
/// Build the absolute path to the snapshot file for the provided disk ID. /// Build the absolute path to the snapshot file for the provided disk ID.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub fn snapshot_path(root: &Path, disk_id: &str) -> PathBuf { pub fn snapshot_path(root: &Path, disk_id: &str) -> PathBuf {
data_usage_dir(root).join(snapshot_file_name(disk_id)) data_usage_dir(root).join(snapshot_file_name(disk_id))
} }
/// Read a snapshot from disk if it exists. /// Read a snapshot from disk if it exists.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result<Option<LocalUsageSnapshot>> { pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result<Option<LocalUsageSnapshot>> {
let path = snapshot_path(root, disk_id); let path = snapshot_path(root, disk_id);
match fs::read(&path).await { match fs::read(&path).await {
@@ -156,7 +138,6 @@ pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result<Option<LocalUsa
} }
/// Persist a snapshot to disk, creating directories as needed and overwriting any existing file. /// Persist a snapshot to disk, creating directories as needed and overwriting any existing file.
#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")]
pub async fn write_snapshot(root: &Path, disk_id: &str, snapshot: &LocalUsageSnapshot) -> Result<()> { pub async fn write_snapshot(root: &Path, disk_id: &str, snapshot: &LocalUsageSnapshot) -> Result<()> {
let dir = data_usage_dir(root); let dir = data_usage_dir(root);
fs::create_dir_all(&dir).await.map_err(Error::other)?; fs::create_dir_all(&dir).await.map_err(Error::other)?;
+105 -136
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
// #730: scanner/data-usage state is partially migrated and still owns staged cache helpers. // #730: scanner/data-usage state is partially migrated and still owns staged cache helpers.
#![allow(dead_code)]
pub mod local_snapshot; pub mod local_snapshot;
@@ -33,8 +34,8 @@ use crate::{
pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path}; pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
use rustfs_data_usage::{ use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
observed_data_usage_is_newer, VersionsHistogram, observed_data_usage_is_newer,
}; };
use rustfs_io_metrics::record_system_path_failure; use rustfs_io_metrics::record_system_path_failure;
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR;
@@ -54,6 +55,7 @@ use tracing::{debug, error, info, instrument};
// Data usage storage constants // Data usage storage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json"; const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30; const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024; const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024;
@@ -311,6 +313,11 @@ lazy_static::lazy_static! {
LEGACY_DATA_USAGE_OBJECT_NAME LEGACY_DATA_USAGE_OBJECT_NAME
); );
static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str());
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR,
DATA_USAGE_BLOOM_NAME
);
pub static ref DATA_COMPRESSION_TOTAL_NAME_PATH: String = format!("{}{}{}", pub static ref DATA_COMPRESSION_TOTAL_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX, crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR, SLASH_SEPARATOR,
@@ -851,10 +858,6 @@ async fn resolve_loaded_snapshot_pair_with_source(
} }
} }
#[allow(
dead_code,
reason = "primary/backup snapshot fallback asserted by this file's tests (backlog#1823)"
)]
async fn resolve_loaded_snapshot( async fn resolve_loaded_snapshot(
primary: Result<Vec<u8>, Error>, primary: Result<Vec<u8>, Error>,
backup: impl Future<Output = Result<Vec<u8>, Error>>, backup: impl Future<Output = Result<Vec<u8>, Error>>,
@@ -1184,10 +1187,6 @@ pub async fn invalidate_admin_data_usage_snapshot_cache() {
} }
/// Aggregate usage information from local disk snapshots. /// Aggregate usage information from local disk snapshots.
#[allow(
dead_code,
reason = "reached only through aggregate_local_snapshots, which has no caller (backlog#1823)"
)]
fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) { fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
if let Some(update) = snapshot.last_update if let Some(update) = snapshot.last_update
&& latest_update.is_none_or(|current| update > current) && latest_update.is_none_or(|current| update > current)
@@ -1221,10 +1220,6 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh
} }
} }
#[allow(
dead_code,
reason = "entry point of the local usage-snapshot feature, which has had no caller since it landed in #5307 (backlog#1823)"
)]
pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskUsageStatus>, DataUsageInfo), Error> { pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskUsageStatus>, DataUsageInfo), Error> {
let mut aggregated = DataUsageInfo::default(); let mut aggregated = DataUsageInfo::default();
let mut latest_update: Option<SystemTime> = None; let mut latest_update: Option<SystemTime> = None;
@@ -1360,7 +1355,7 @@ impl BucketUsageAccumulator {
return Ok(()); return Ok(());
} }
let object_size = quota_object_size(object)?; let object_size = object.size.max(0) as u64;
self.current_live_versions = self.current_live_versions.saturating_add(1); self.current_live_versions = self.current_live_versions.saturating_add(1);
self.size_histogram.add(object_size); self.size_histogram.add(object_size);
self.total_size = self.total_size.saturating_add(object_size); self.total_size = self.total_size.saturating_add(object_size);
@@ -1390,31 +1385,6 @@ impl BucketUsageAccumulator {
} }
} }
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
let persisted_part_size = if object.parts.is_empty() {
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
} else {
object.parts.iter().try_fold(0_u64, |total, part| {
// Compressed streaming objects persist -1 when the transformed
// part size is unknown. The physical part size remains a valid
// quota floor; reject only non-negative values that overflow.
let actual_size = if part.actual_size < 0 {
if object.is_compressed() {
0
} else {
return Err(Error::PartMissingOrCorrupt);
}
} else {
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
};
let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?);
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
})?
};
Ok(logical_size.max(persisted_part_size))
}
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>; type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> { pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
@@ -1772,6 +1742,11 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne
entry.pending_scanner_position = None; entry.pending_scanner_position = None;
} }
/// Fast in-memory increment for immediate quota consistency.
pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) {
record_bucket_object_write_memory(bucket, None, size_increment).await;
}
/// Fast in-memory update for successful object deletes. /// Fast in-memory update for successful object deletes.
pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
ensure_bucket_usage_cached(bucket).await; ensure_bucket_usage_cached(bucket).await;
@@ -1814,6 +1789,11 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) {
entry.pending_scanner_position = None; entry.pending_scanner_position = None;
} }
/// Fast in-memory decrement for immediate quota consistency
pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await;
}
/// Get bucket usage from the authoritative cache for this topology. /// Get bucket usage from the authoritative cache for this topology.
async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> { async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
let store = runtime_sources::object_store_handle()?; let store = runtime_sources::object_store_handle()?;
@@ -2008,6 +1988,91 @@ pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageIn
apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await; apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await;
} }
/// Sync memory cache with backend data (called by scanner)
pub async fn sync_memory_cache_with_backend() -> Result<(), Error> {
if let Some(store) = runtime_sources::object_store_handle() {
match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage_info) => {
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
Err(e) => {
debug!("Failed to sync memory cache with backend: {}", e);
}
}
}
Ok(())
}
/// Create a data usage cache entry from size summary
pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry {
let mut entry = DataUsageEntry::default();
entry.add_sizes(summary);
entry
}
/// Convert data usage cache to DataUsageInfo
pub fn cache_to_data_usage_info(
cache: &DataUsageCache,
path: &str,
buckets: &[crate::storage_api_contracts::bucket::BucketInfo],
) -> DataUsageInfo {
let e = match cache.find(path) {
Some(e) => e,
None => return DataUsageInfo::default(),
};
let flat = cache.flatten(&e);
let mut buckets_usage = HashMap::new();
for bucket in buckets.iter() {
let e = match cache.find(&bucket.name) {
Some(e) => e,
None => continue,
};
let flat = cache.flatten(&e);
let mut bui = BucketUsageInfo {
size: flat.size as u64,
versions_count: flat.versions as u64,
objects_count: flat.objects as u64,
delete_markers_count: flat.delete_markers as u64,
object_size_histogram: flat.obj_sizes.to_map(),
object_versions_histogram: flat.obj_versions.to_map(),
..Default::default()
};
if let Some(rs) = &flat.replication_stats {
bui.replica_size = rs.replica_size;
bui.replica_count = rs.replica_count;
for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert(
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
replication_failed_size: stat.failed_size,
replication_pending_count: stat.pending_count,
replication_failed_count: stat.failed_count,
replicated_count: stat.replicated_count,
..Default::default()
},
);
}
}
buckets_usage.insert(bucket.name.clone(), bui);
}
DataUsageInfo {
last_update: cache.info.last_update,
objects_total_count: flat.objects as u64,
versions_total_count: flat.versions as u64,
delete_markers_total_count: flat.delete_markers as u64,
objects_total_size: flat.size as u64,
buckets_count: e.children.len() as u64,
buckets_usage,
..Default::default()
}
}
// Helper functions for DataUsageCache operations // Helper functions for DataUsageCache operations
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> { pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
@@ -3059,102 +3124,6 @@ mod tests {
assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1)); assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1));
} }
#[test]
fn bucket_usage_uses_the_larger_of_logical_and_physical_size() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string());
let object = ObjectInfo {
name: "compressed".to_string(),
size: 128,
user_defined: Arc::new(metadata),
..Default::default()
};
let mut usage = BucketUsageAccumulator::default();
usage
.record("bucket", &object)
.expect("valid compressed metadata should be counted");
assert_eq!(usage.finish().size, 4096);
let mut framed_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut framed_metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
let framed = ObjectInfo {
name: "framed".to_string(),
size: 17,
user_defined: Arc::new(framed_metadata),
..Default::default()
};
assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17);
let legacy_compressed_part = ObjectInfo {
name: "legacy-compressed-part".to_string(),
size: 1,
user_defined: Arc::new((*framed.user_defined).clone()),
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 1,
actual_size: -1,
..Default::default()
}]),
..Default::default()
};
assert_eq!(
quota_object_size(&legacy_compressed_part).expect("unknown compressed part size is a valid sentinel"),
1
);
let uncompressed_negative_part = ObjectInfo {
name: "uncompressed-negative-part".to_string(),
size: 1,
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 1,
actual_size: -1,
..Default::default()
}]),
..Default::default()
};
assert!(matches!(quota_object_size(&uncompressed_negative_part), Err(Error::PartMissingOrCorrupt)));
let mut corrupt_metadata = (*object.user_defined).clone();
rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string());
let corrupt = ObjectInfo {
user_defined: Arc::new(corrupt_metadata),
..object
};
assert!(matches!(quota_object_size(&corrupt), Err(Error::PartMissingOrCorrupt)));
let mut poisoned_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut poisoned_metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
let poisoned = ObjectInfo {
name: "legacy-swift-metadata".to_string(),
size: 4096,
user_defined: Arc::new(poisoned_metadata),
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 4096,
actual_size: 4096,
..Default::default()
}]),
..Default::default()
};
assert_eq!(
quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"),
4096
);
}
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() { async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
-37
View File
@@ -190,17 +190,6 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned"; pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
@@ -562,32 +551,6 @@ mod tests {
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned"); assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
"data_read_inline_body_verify"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
"data_read_inline_identity_mismatch"
);
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
"data_read_inline_missing_payload"
);
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
"data_read_inline_missing_shard"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
"data_read_inline_transformed"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
+7 -36
View File
@@ -241,40 +241,6 @@ pub fn get_drive_list_dir_timeout() -> Duration {
) )
} }
pub(crate) trait DiskStoreRenameDataExt {
async fn rename_data_borrowed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
}
impl DiskStoreRenameDataExt for LocalDiskWrapper {
async fn rename_data_borrowed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
self.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
self.disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.await
},
get_max_timeout_duration(),
)
.await
}
}
pub fn get_drive_walkdir_timeout() -> Duration { pub fn get_drive_walkdir_timeout() -> Duration {
get_drive_timeout_duration( get_drive_timeout_duration(
rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
@@ -2019,8 +1985,13 @@ impl DiskAPI for LocalDiskWrapper {
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) self.track_disk_health_mutation(
.await "rename_data",
DiskMetricMutation::Write,
|| async { self.disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await },
get_max_timeout_duration(),
)
.await
} }
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> { async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
+25 -209
View File
@@ -27,18 +27,17 @@ use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo,
DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize,
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET,
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, conv_part_err_to_int,
endpoint::Endpoint, endpoint::Endpoint,
error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3, format::FormatV3,
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
is_quota_mutation_fence_path, os, os,
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
quota_mutation_fence_path,
}; };
use crate::erasure::coding::{self, bitrot_verify}; use crate::erasure::coding::{self, bitrot_verify};
use crate::runtime::sources as runtime_sources; use crate::runtime::sources as runtime_sources;
@@ -61,7 +60,9 @@ use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt::Debug; use std::fmt::Debug;
use std::io::{Error as IoError, SeekFrom}; use std::io::{Error as IoError, SeekFrom};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; #[cfg(target_os = "linux")]
use std::sync::atomic::AtomicBool;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
use std::time::Duration; use std::time::Duration;
use std::{ use std::{
@@ -2028,17 +2029,14 @@ static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex<Option<(Strin
#[cfg(test)] #[cfg(test)]
type InlinePreparationHook = Box<dyn FnOnce() + Send>; type InlinePreparationHook = Box<dyn FnOnce() + Send>;
#[cfg(test)] #[cfg(test)]
type RenameDataPublicationHookKey = (PathBuf, String, String);
#[cfg(test)]
static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> = static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
#[cfg(test)] #[cfg(test)]
static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> = static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
#[cfg(test)] #[cfg(test)]
static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock< static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
std::sync::Mutex<HashMap<RenameDataPublicationHookKey, InlinePreparationHook>>, std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
#[cfg(test)] #[cfg(test)]
static OWNED_FILE_WRITE_BEFORE_OPEN: std::sync::LazyLock<std::sync::Mutex<HashMap<PathBuf, InlinePreparationHook>>> = static OWNED_FILE_WRITE_BEFORE_OPEN: std::sync::LazyLock<std::sync::Mutex<HashMap<PathBuf, InlinePreparationHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
@@ -2110,11 +2108,11 @@ fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + S
} }
#[cfg(test)] #[cfg(test)]
fn set_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str, hook: impl FnOnce() + Send + 'static) { fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
RENAME_DATA_AFTER_FIRST_PUBLICATION RENAME_DATA_AFTER_FIRST_PUBLICATION
.lock() .lock()
.expect("test publication hook lock should not be poisoned") .expect("test publication hook lock should not be poisoned")
.insert((root.to_path_buf(), dst_volume.to_string(), dst_path.to_string()), Box::new(hook)); .insert(dst_path.to_string(), Box::new(hook));
} }
#[cfg(test)] #[cfg(test)]
@@ -2266,11 +2264,11 @@ fn run_inline_before_file_sync_admission(dst_path: &str) {
} }
#[cfg(test)] #[cfg(test)]
fn run_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str) { fn run_rename_data_after_first_publication(dst_path: &str) {
let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION
.lock() .lock()
.expect("test publication hook lock should not be poisoned") .expect("test publication hook lock should not be poisoned")
.remove(&(root.to_path_buf(), dst_volume.to_string(), dst_path.to_string())); .remove(dst_path);
if let Some(hook) = hook { if let Some(hook) = hook {
hook(); hook();
} }
@@ -2368,6 +2366,9 @@ async fn remove_dst_base_before_commit(
#[cfg(not(test))] #[cfg(not(test))]
fn run_inline_preparation_before_backup(_dst_path: &str) {} fn run_inline_preparation_before_backup(_dst_path: &str) {}
#[cfg(not(test))]
fn run_rename_data_after_first_publication(_dst_path: &str) {}
#[cfg(not(test))] #[cfg(not(test))]
fn should_fail_after_delete_data_staged(_path: &str) -> bool { fn should_fail_after_delete_data_staged(_path: &str) -> bool {
false false
@@ -4755,25 +4756,6 @@ struct SnapshotLeaseEntry {
tokens: HashSet<SnapshotLeaseToken>, tokens: HashSet<SnapshotLeaseToken>,
pending_delete: Option<DeleteOptions>, pending_delete: Option<DeleteOptions>,
deleting: bool, deleting: bool,
mutation_fence: Option<Arc<QuotaMutationFenceState>>,
}
#[derive(Default)]
struct QuotaMutationFenceState {
revoked: AtomicBool,
running: AtomicUsize,
notify: Notify,
}
struct QuotaMutationFenceClaim {
state: Arc<QuotaMutationFenceState>,
}
impl Drop for QuotaMutationFenceClaim {
fn drop(&mut self) {
self.state.running.fetch_sub(1, Ordering::AcqRel);
self.state.notify.notify_waiters();
}
} }
#[derive(Default)] #[derive(Default)]
@@ -7338,34 +7320,6 @@ fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
} }
impl LocalDisk { impl LocalDisk {
async fn claim_quota_mutation_fence(
&self,
volume: &str,
path: &str,
token: SnapshotLeaseToken,
) -> Result<Arc<QuotaMutationFenceClaim>> {
let key = SnapshotLeaseKey {
volume: RUSTFS_META_BUCKET.to_string(),
path: quota_mutation_fence_path(volume, path),
};
let state = {
let registry = self.snapshot_leases.lock().await;
let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?;
let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?;
if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) {
return Err(DiskError::FileNotFound);
}
state.running.fetch_add(1, Ordering::AcqRel);
Arc::clone(state)
};
if state.revoked.load(Ordering::Acquire) {
state.running.fetch_sub(1, Ordering::AcqRel);
state.notify.notify_waiters();
return Err(DiskError::FileNotFound);
}
Ok(Arc::new(QuotaMutationFenceClaim { state }))
}
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> { async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
let path = format!("{object}/{data_dir}"); let path = format!("{object}/{data_dir}");
let data_path = self.io_get_object_path(volume, &path)?; let data_path = self.io_get_object_path(volume, &path)?;
@@ -8689,41 +8643,17 @@ impl DiskAPI for LocalDisk {
&self, &self,
src_volume: &str, src_volume: &str,
src_path: &str, src_path: &str,
fi: FileInfo, mut fi: FileInfo,
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
crate::hp_guard!("LocalDisk::rename_data"); crate::hp_guard!("LocalDisk::rename_data");
let mut fi = fi;
// A non-force DeleteBucket must not remove a directory while a local // A non-force DeleteBucket must not remove a directory while a local
// object commit is publishing into it. The peer's empty scan remains // object commit is publishing into it. The peer's empty scan remains
// optimistic; this lease establishes the local commit/delete order and // optimistic; this lease establishes the local commit/delete order and
// remains owned by any blocking syscall that outlives async cancellation. // remains owned by any blocking syscall that outlives async cancellation.
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
let quota_fence_token =
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
Some(value) => {
let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?;
Some(SnapshotLeaseToken::from_slice(token.as_bytes())?)
}
None if rustfs_utils::http::metadata_compat::contains_key_str(
&fi.metadata,
QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
) =>
{
return Err(DiskError::FileCorrupt);
}
None => None,
};
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX);
let quota_fence_claim = match quota_fence_token {
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
None => None,
};
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
if let Some(claim) = quota_fence_claim {
mutation_lease.attach_external_guard(claim);
}
if fi.is_legacy_indexed_delete_marker() { if fi.is_legacy_indexed_delete_marker() {
fi.erasure.index = 0; fi.erasure.index = 0;
} }
@@ -9016,9 +8946,8 @@ impl DiskAPI for LocalDisk {
.await?; .await?;
return Err(err); return Err(err);
} }
#[cfg(test)]
if has_data_dir_path.is_some() { if has_data_dir_path.is_some() {
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); run_rename_data_after_first_publication(dst_path);
} }
// Crash-consistency injection: hard power loss after the data dir // Crash-consistency injection: hard power loss after the data dir
@@ -9451,8 +9380,7 @@ impl DiskAPI for LocalDisk {
let _ = remove_file_if_exists(staged_backup); let _ = remove_file_if_exists(staged_backup);
return Err(err); return Err(err);
} }
#[cfg(test)] run_rename_data_after_first_publication(dst_path);
run_rename_data_after_first_publication(&self.root, dst_volume, dst_path);
if sync { if sync {
file_sync_admission = Some( file_sync_admission = Some(
os::acquire_file_sync_admission(self.file_sync_permits.clone()) os::acquire_file_sync_admission(self.file_sync_permits.clone())
@@ -9715,26 +9643,11 @@ impl DiskAPI for LocalDisk {
} }
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> { async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
let file_path = self.io_get_object_path(volume, path)?;
let key = SnapshotLeaseKey { let key = SnapshotLeaseKey {
volume: volume.to_string(), volume: volume.to_string(),
path: path.to_string(), path: path.to_string(),
}; };
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
let mut registry = self.snapshot_leases.lock().await;
let entry = registry.entries.entry(key).or_default();
let state = entry
.mutation_fence
.get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default()));
if state.revoked.load(Ordering::Acquire) {
return Err(DiskError::FileNotFound);
}
let token = SnapshotLeaseToken::new();
entry.tokens.insert(token);
return Ok(token);
}
let file_path = self.io_get_object_path(volume, path)?;
let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await;
let token = { let token = {
let mut registry = self.snapshot_leases.lock().await; let mut registry = self.snapshot_leases.lock().await;
if registry.entries.get(&key).is_some_and(|entry| entry.deleting) { if registry.entries.get(&key).is_some_and(|entry| entry.deleting) {
@@ -9762,48 +9675,6 @@ impl DiskAPI for LocalDisk {
volume: volume.to_string(), volume: volume.to_string(),
path: path.to_string(), path: path.to_string(),
}; };
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
if !token.is_revoke_all() {
let mut registry = self.snapshot_leases.lock().await;
let Some(entry) = registry.entries.get_mut(&key) else {
return Ok(());
};
entry.tokens.remove(&token);
let removable = entry.tokens.is_empty()
&& entry
.mutation_fence
.as_ref()
.is_none_or(|state| state.running.load(Ordering::Acquire) == 0);
if removable {
registry.entries.remove(&key);
}
return Ok(());
}
let state = {
let mut registry = self.snapshot_leases.lock().await;
let Some(entry) = registry.entries.get_mut(&key) else {
return Ok(());
};
let Some(state) = entry.mutation_fence.as_ref().cloned() else {
registry.entries.remove(&key);
return Ok(());
};
state.revoked.store(true, Ordering::Release);
entry.tokens.clear();
state
};
loop {
let notified = state.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if state.running.load(Ordering::Acquire) == 0 {
break;
}
notified.await;
}
self.snapshot_leases.lock().await.entries.remove(&key);
return Ok(());
}
let opts = { let opts = {
let mut registry = self.snapshot_leases.lock().await; let mut registry = self.snapshot_leases.lock().await;
let Some(entry) = registry.entries.get_mut(&key) else { let Some(entry) = registry.entries.get_mut(&key) else {
@@ -10582,19 +10453,6 @@ impl DiskAPI for LocalDisk {
} }
} }
impl LocalDisk {
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
<Self as DiskAPI>::rename_data(self, src_volume, src_path, fi.clone(), dst_volume, dst_path).await
}
}
async fn wait_for_startup_cleanup_signal( async fn wait_for_startup_cleanup_signal(
startup_cleanup_ready: &AtomicU32, startup_cleanup_ready: &AtomicU32,
startup_cleanup_notify: &Notify, startup_cleanup_notify: &Notify,
@@ -13149,7 +13007,7 @@ mod test {
let replacement_staging_parent_for_hook = replacement_staging_parent.clone(); let replacement_staging_parent_for_hook = replacement_staging_parent.clone();
let staged_metadata_for_hook = staged_metadata.clone(); let staged_metadata_for_hook = staged_metadata.clone();
let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone();
set_rename_data_after_first_publication(&disk.root, bucket, object, move || { set_rename_data_after_first_publication(object, move || {
std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook)
.expect_err("the destination object identity must remain pinned until xl.meta commits"); .expect_err("the destination object identity must remain pinned until xl.meta commits");
std::fs::rename(&staging_parent_for_hook, &replacement_staging_parent_for_hook) std::fs::rename(&staging_parent_for_hook, &replacement_staging_parent_for_hook)
@@ -13416,7 +13274,7 @@ mod test {
let replacement_dir_for_hook = replacement_dir.clone(); let replacement_dir_for_hook = replacement_dir.clone();
let staged_metadata_for_hook = staged_metadata.clone(); let staged_metadata_for_hook = staged_metadata.clone();
let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone();
set_rename_data_after_first_publication(&disk.root, bucket, object, move || { set_rename_data_after_first_publication(object, move || {
std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook)
.expect_err("the destination object identity must remain pinned after publishing its rollback backup"); .expect_err("the destination object identity must remain pinned after publishing its rollback backup");
std::fs::rename(&staged_metadata_for_hook, &replacement_staged_metadata_for_hook) std::fs::rename(&staged_metadata_for_hook, &replacement_staged_metadata_for_hook)
@@ -13782,7 +13640,7 @@ mod test {
let (entered_tx, entered_rx) = mpsc::channel(); let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel();
set_rename_data_after_first_publication(&disk.root, bucket, object, move || { set_rename_data_after_first_publication(object, move || {
entered_tx.send(()).expect("signal first publication"); entered_tx.send(()).expect("signal first publication");
release_rx.recv().expect("wait while delete_volume is blocked"); release_rx.recv().expect("wait while delete_volume is blocked");
}); });
@@ -14376,7 +14234,7 @@ mod test {
let (published_tx, published_rx) = mpsc::channel(); let (published_tx, published_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel();
set_rename_data_after_first_publication(&disk.root, bucket, object, move || { set_rename_data_after_first_publication(object, move || {
published_tx.send(()).expect("signal backup publication"); published_tx.send(()).expect("signal backup publication");
release_rx.recv().expect("wait for lock-order assertion"); release_rx.recv().expect("wait for lock-order assertion");
}); });
@@ -19031,48 +18889,6 @@ mod test {
assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound))); assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
} }
#[tokio::test]
async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() {
use tempfile::tempdir;
let root_dir = tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
let bucket = "quota-fence-volume";
let object = "object";
let fence_path = quota_mutation_fence_path(bucket, object);
let token = disk
.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path)
.await
.expect("quota mutation token should be prepared");
let claim = disk
.claim_quota_mutation_fence(bucket, object, token)
.await
.expect("prepared token should be claimable");
let release_disk = Arc::clone(&disk);
let mut release = tokio::spawn(async move {
release_disk
.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all())
.await
});
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(),
"revoke must wait until an already claimed mutation has finished"
);
drop(claim);
tokio::time::timeout(Duration::from_secs(1), release)
.await
.expect("revoke should wake after the final claim drops")
.expect("revoke task should not panic")
.expect("revoke should succeed");
assert!(matches!(
disk.claim_quota_mutation_fence(bucket, object, token).await,
Err(DiskError::FileNotFound)
));
}
#[tokio::test] #[tokio::test]
async fn delete_version_keeps_later_part_until_snapshot_release() { async fn delete_version_keeps_later_part_until_snapshot_release() {
use tempfile::tempdir; use tempfile::tempdir;
+4 -63
View File
@@ -55,7 +55,6 @@ pub fn part_transaction_path(part_path: &str) -> String {
use crate::cluster::rpc::RemoteDisk; use crate::cluster::rpc::RemoteDisk;
use crate::cluster::rpc::build_internode_data_transport_from_env; use crate::cluster::rpc::build_internode_data_transport_from_env;
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::disk_store::LocalDiskWrapper; use crate::disk::disk_store::LocalDiskWrapper;
use crate::disk::health_state::RuntimeDriveHealthState; use crate::disk::health_state::RuntimeDriveHealthState;
use crate::disk::local::ScanGuard; use crate::disk::local::ScanGuard;
@@ -73,28 +72,6 @@ use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid; use uuid::Uuid;
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String {
use sha2::{Digest, Sha256};
let mut input = Vec::with_capacity(bucket.len() + object.len() + 1);
input.extend_from_slice(bucket.as_bytes());
input.push(0);
input.extend_from_slice(object.as_bytes());
let digest = Sha256::digest(input);
format!(
"{QUOTA_MUTATION_FENCE_PREFIX}{}",
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
)
}
pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool {
path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX)
.is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()))
}
pub type DiskStore = Arc<Disk>; pub type DiskStore = Arc<Disk>;
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>; pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
@@ -119,20 +96,6 @@ impl SnapshotLeaseToken {
pub fn as_bytes(&self) -> &[u8; 16] { pub fn as_bytes(&self) -> &[u8; 16] {
self.0.as_bytes() self.0.as_bytes()
} }
pub(crate) fn as_uuid(self) -> Uuid {
self.0
}
#[doc(hidden)]
pub fn revoke_all() -> Self {
Self(Uuid::nil())
}
#[doc(hidden)]
pub fn is_revoke_all(self) -> bool {
self.0.is_nil()
}
} }
impl Default for SnapshotLeaseToken { impl Default for SnapshotLeaseToken {
@@ -435,8 +398,10 @@ impl DiskAPI for Disk {
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) match self {
.await Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
}
} }
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
@@ -666,30 +631,6 @@ impl DiskAPI for Disk {
} }
} }
impl Disk {
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.await
}
Disk::Remote(remote_disk) => {
remote_disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.await
}
}
}
}
impl Disk { impl Disk {
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> { pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
match self { match self {
-9
View File
@@ -306,20 +306,12 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
pub(crate) struct NamespaceMutationLease { pub(crate) struct NamespaceMutationLease {
_namespace_guard: OwnedMutexGuard<()>, _namespace_guard: OwnedMutexGuard<()>,
_volume_guard: Option<OwnedRwLockReadGuard<()>>, _volume_guard: Option<OwnedRwLockReadGuard<()>>,
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
}
impl NamespaceMutationLease {
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
*self.external_guard.lock() = Some(guard);
}
} }
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> { async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
Arc::new(NamespaceMutationLease { Arc::new(NamespaceMutationLease {
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await, _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
_volume_guard: None, _volume_guard: None,
external_guard: Mutex::new(None),
}) })
} }
@@ -335,7 +327,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
Arc::new(NamespaceMutationLease { Arc::new(NamespaceMutationLease {
_namespace_guard: namespace_guard, _namespace_guard: namespace_guard,
_volume_guard: Some(volume_guard), _volume_guard: Some(volume_guard),
external_guard: Mutex::new(None),
}) })
} }
+69
View File
@@ -1075,6 +1075,9 @@ pub struct GenericError {
#[derive(Debug, thiserror::Error, PartialEq, Eq)] #[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum ObjectApiError { pub enum ObjectApiError {
#[error("BackendDown")]
BackendDown(String),
#[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)] #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)]
InvalidObjectState(GenericError), InvalidObjectState(GenericError),
} }
@@ -1091,6 +1094,72 @@ pub struct ErrorResponse {
pub host_id: String, pub host_id: String,
} }
pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error {
let mut bucket = "";
let mut object = "";
let mut version_id = "";
if !params.is_empty() {
bucket = params[0];
}
if params.len() >= 2 {
object = params[1];
}
if params.len() >= 3 {
version_id = params[2];
}
if is_network_or_host_down(&err.to_string(), false) {
return std::io::Error::other(ObjectApiError::BackendDown(format!("{err}")));
}
let err_ = std::io::Error::other(err.to_string());
let r_err = err;
let err;
let bucket = bucket.to_string();
let object = object.to_string();
let version_id = version_id.to_string();
match r_err.code {
S3ErrorCode::BucketNotEmpty => {
err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string());
}
S3ErrorCode::InvalidBucketName => {
err = std::io::Error::other(StorageError::BucketNameInvalid(bucket));
}
S3ErrorCode::InvalidPart => {
err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */));
}
S3ErrorCode::NoSuchBucket => {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
S3ErrorCode::NoSuchKey => {
if !object.is_empty() {
err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));
} else {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
}
S3ErrorCode::NoSuchVersion => {
if !object.is_empty() {
err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id);
} else {
err = std::io::Error::other(StorageError::BucketNotFound(bucket));
}
}
S3ErrorCode::AccessDenied => {
err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object));
}
S3ErrorCode::NoSuchUpload => {
err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id));
}
_ => {
err = err_;
}
}
err
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
-4
View File
@@ -20,10 +20,6 @@ use std::sync::atomic::AtomicI64;
/// this type never grew past its counter. `total_events` is read by the /// this type never grew past its counter. `total_events` is read by the
/// notifier's log line but nothing increments it, so that field reports zero. /// notifier's log line but nothing increments it, so that field reports zero.
#[derive(Default)] #[derive(Default)]
#[allow(
dead_code,
reason = "held only by the dead ecstore EventNotifier; see services/event_notification.rs (backlog#1823)"
)]
pub struct TargetList { pub struct TargetList {
pub total_events: AtomicI64, pub total_events: AtomicI64,
} }
-22
View File
@@ -31,13 +31,6 @@ pub const ENV_DISK_COMPRESSION_MIME_TYPES: &str = "RUSTFS_COMPRESSION_MIME_TYPES
// Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar") // Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar")
pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS"; pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS";
// Environment variable to additionally enable disk compression for multipart uploads.
// Default off: nodes from before the resumable decompressor fix fail transient reads of
// compressed objects, so multipart compression stays dark until the operator confirms the
// fleet has converged on a fixed build.
// RUSTFS_COMPAT_TODO(multipart-compression-default-off-window): staged rollout switch for restored multipart compression, flipping the default to enabled on retirement. Remove after the minimum supported direct-upgrade release ships the resumable DecompressReader.
pub const ENV_DISK_COMPRESSION_MULTIPART_ENABLED: &str = "RUSTFS_COMPRESSION_MULTIPART_ENABLED";
pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin"; pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin";
pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream"; pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream";
@@ -178,21 +171,6 @@ pub fn is_disk_compression_enabled() -> bool {
DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config).enabled DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config).enabled
} }
// Parsed once at first use, mirroring DISK_COMPRESSION_CONFIG.
static MULTIPART_DISK_COMPRESSION_ENABLED: OnceLock<bool> = OnceLock::new();
/// Whether multipart uploads may advertise disk compression. Requires the
/// regular disk-compression gates to pass as well; this is the staged-rollout
/// switch that keeps multipart compression dark during rolling upgrades from
/// builds whose decompressor was not yet resumable.
pub fn is_multipart_disk_compression_enabled() -> bool {
*MULTIPART_DISK_COMPRESSION_ENABLED.get_or_init(|| {
env::var(ENV_DISK_COMPRESSION_MULTIPART_ENABLED)
.map(|s| matches!(s.to_ascii_lowercase().as_str(), "true" | "on" | "1"))
.unwrap_or(false)
})
}
fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool { fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool {
// Check if disk compression is enabled (read once at first use, then fixed for process lifetime) // Check if disk compression is enabled (read once at first use, then fixed for process lifetime)
if !config.enabled { if !config.enabled {
+1 -1
View File
@@ -22,7 +22,7 @@ use crate::bucket::replication::{
use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass; use crate::config::storageclass;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::io_support::rio::{HardLimitReader, HashReader}; use crate::io_support::rio::{HashReader, LimitReader};
use crate::storage_api_contracts::{ use crate::storage_api_contracts::{
lifecycle::{ExpirationOptions, TransitionedObject}, lifecycle::{ExpirationOptions, TransitionedObject},
range::HTTPRangeSpec, range::HTTPRangeSpec,
+4 -475
View File
@@ -479,15 +479,7 @@ enum ReadTransform {
}, },
} }
/// How an object's stored bytes must be fetched and transformed to serve a struct ReadPlan {
/// request.
///
/// Public so callers that fetch the stored bytes from somewhere other than the
/// local erasure set — the remote-tier read path — can position their own fetch
/// with [`ReadPlan::storage_offset`] / [`ReadPlan::storage_length`] and then
/// hand the resulting stream to [`ReadPlan::into_object_reader`], instead of
/// reimplementing the transform decisions (rustfs/rustfs#6025).
pub struct ReadPlan {
storage_offset: usize, storage_offset: usize,
storage_length: i64, storage_length: i64,
object_size: i64, object_size: i64,
@@ -495,43 +487,6 @@ pub struct ReadPlan {
} }
impl ReadPlan { impl ReadPlan {
/// Byte offset into the object's **stored** bytes where the fetch must
/// start. Encrypted and compressed objects address their storage in a
/// different coordinate system than the plaintext range the caller asked
/// for, which is exactly the distinction this plan resolves.
pub fn storage_offset(&self) -> usize {
self.storage_offset
}
/// Number of **stored** bytes the fetch must deliver, in the same
/// coordinate system as [`Self::storage_offset`].
pub fn storage_length(&self) -> i64 {
self.storage_length
}
/// Build the plan for a request without consuming a stream, so a caller
/// that has to issue its own positioned fetch can read the offsets first.
pub async fn build_for_request(
rs: Option<HTTPRangeSpec>,
oi: &ObjectInfo,
opts: &ObjectOptions,
h: &HeaderMap<HeaderValue>,
resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<Self> {
Self::build_with_resolver(rs, oi, opts, h, resolver).await
}
/// Wrap `reader` — the stored bytes this plan asked for, already positioned
/// at [`Self::storage_offset`] — in the transforms that turn them into the
/// bytes the caller requested.
pub fn into_object_reader(
self,
reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
oi: &ObjectInfo,
) -> Result<GetObjectReader> {
self.into_reader(reader, oi).map(|(reader, _, _)| reader)
}
#[cfg(test)] #[cfg(test)]
async fn build(rs: Option<HTTPRangeSpec>, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap<HeaderValue>) -> Result<Self> { async fn build(rs: Option<HTTPRangeSpec>, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap<HeaderValue>) -> Result<Self> {
Self::build_with_resolver(rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await Self::build_with_resolver(rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await
@@ -545,17 +500,8 @@ impl ReadPlan {
resolver: Option<&dyn ObjectEncryptionResolver>, resolver: Option<&dyn ObjectEncryptionResolver>,
) -> Result<Self> { ) -> Result<Self> {
let mut rs = rs; let mut rs = rs;
// A part number addresses the object's PLAINTEXT bytes. A restore read
// serves the stored representation instead (see
// [`restore_request_active`]), where that synthesized range would be
// reinterpreted as a storage range and truncate an encrypted or
// compressed payload by exactly its encoding overhead — the copy-back
// then fails its length check partway through
// (rustfs/rustfs#6025). An explicit caller range is already in storage
// coordinates on that path and is still honored.
if let Some(part_number) = opts.part_number if let Some(part_number) = opts.part_number
&& rs.is_none() && rs.is_none()
&& !restore_request_active(opts)
{ {
rs = http_range_spec_from_object_info(oi, part_number); rs = http_range_spec_from_object_info(oi, part_number);
} }
@@ -808,7 +754,7 @@ impl ReadPlan {
} }
} }
} else { } else {
Box::new(HardLimitReader::new(dec_reader, decompressed_length)) Box::new(LimitReader::new(dec_reader, total_plaintext_size))
}; };
let mut object_info = oi.clone(); let mut object_info = oi.clone();
@@ -900,7 +846,7 @@ impl ReadPlan {
)?; )?;
Box::new(ranged_reader) Box::new(ranged_reader)
} else { } else {
Box::new(HardLimitReader::new(decompressed_reader, total_plaintext_size_i64)) Box::new(LimitReader::new(decompressed_reader, total_plaintext_size))
} }
} else if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 { } else if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 {
Box::new(RangedDecompressReader::new( Box::new(RangedDecompressReader::new(
@@ -910,7 +856,7 @@ impl ReadPlan {
total_plaintext_size, total_plaintext_size,
)?) )?)
} else { } else {
Box::new(HardLimitReader::new(decrypted_reader, total_plaintext_size_i64)) Box::new(LimitReader::new(decrypted_reader, total_plaintext_size))
}; };
let mut object_info = oi.clone(); let mut object_info = oi.clone();
@@ -1781,423 +1727,6 @@ mod tests {
assert_eq!(actual, b"fghijkl"); assert_eq!(actual, b"fghijkl");
} }
/// Compresses one multipart part exactly like the write path does
/// (`WritePlan::with_compression` wraps each part in its own
/// `compression_reader`), returning the on-disk bytes and the storage-format
/// compression index.
async fn compressed_part_fixture(data: &[u8]) -> (Vec<u8>, Option<Bytes>) {
use crate::io_support::rio::TryGetIndex as _;
let mut compressor =
crate::io_support::rio::compression_reader(Cursor::new(data.to_vec()), CompressionAlgorithm::default(), false);
let mut compressed = Vec::new();
compressor.read_to_end(&mut compressed).await.expect("compress part stream");
let index = compressor
.try_get_index()
.map(crate::io_support::rio::compression_index_storage_bytes);
(compressed, index)
}
struct CompressedMultipartFixture {
object_info: ObjectInfo,
stored: Vec<u8>,
plaintext: Vec<u8>,
}
/// Builds the on-disk representation of a compressed multipart object: each
/// part is an independent compressed stream and the storage layer serves
/// their concatenation.
async fn compressed_multipart_fixture(part_sizes: &[usize]) -> CompressedMultipartFixture {
let pattern = b"compressed multipart read path fixture data ";
let mut plaintext = Vec::new();
let mut stored = Vec::new();
let mut parts = Vec::with_capacity(part_sizes.len());
for (i, part_size) in part_sizes.iter().enumerate() {
let mut part_plaintext = Vec::with_capacity(*part_size);
while part_plaintext.len() < *part_size {
part_plaintext.extend_from_slice(pattern);
part_plaintext.push(i as u8);
}
part_plaintext.truncate(*part_size);
let (compressed, index) = compressed_part_fixture(&part_plaintext).await;
parts.push(ObjectPartInfo {
number: i + 1,
size: compressed.len(),
actual_size: *part_size as i64,
index,
..Default::default()
});
stored.extend_from_slice(&compressed);
plaintext.extend_from_slice(&part_plaintext);
}
let mut user_defined = HashMap::new();
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()),
);
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
let object_info = ObjectInfo {
bucket: "test-bucket".to_string(),
name: "compressed-multipart".to_string(),
size: stored.len() as i64,
etag: Some(format!("6bcf86bed8807b8e78f0fc6e0a53079d-{}", part_sizes.len())),
parts: Arc::new(parts),
user_defined: Arc::new(user_defined),
..Default::default()
};
CompressedMultipartFixture {
object_info,
stored,
plaintext,
}
}
/// Plans the read once to learn the storage window, then serves exactly that
/// window — mirroring how `set_disk` feeds the erasure read into the
/// returned reader.
async fn read_compressed_multipart(
fixture: &CompressedMultipartFixture,
rs: Option<HTTPRangeSpec>,
opts: &ObjectOptions,
) -> Vec<u8> {
let headers = HeaderMap::new();
let (_, offset, length) =
GetObjectReader::new(Box::new(Cursor::new(Vec::new())), rs.clone(), &fixture.object_info, opts, &headers)
.await
.expect("plan compressed multipart read");
let end = offset + usize::try_from(length).expect("storage window length must be non-negative");
assert!(
end <= fixture.stored.len(),
"planned storage window {offset}..{end} exceeds stored stream of {} bytes",
fixture.stored.len()
);
let window = fixture.stored[offset..end].to_vec();
let (mut reader, replay_offset, replay_length) =
GetObjectReader::new(Box::new(Cursor::new(window)), rs, &fixture.object_info, opts, &headers)
.await
.expect("build compressed multipart reader");
assert_eq!((replay_offset, replay_length), (offset, length), "read plan must be deterministic");
reader.read_all().await.expect("read compressed multipart stream")
}
/// Byte pattern with a 2 KiB period: it compresses extremely well while
/// looking nothing like ASCII fixtures. Mirrors the e2e generator that
/// exposed a truncated full GET on high-ratio multipart payloads.
fn high_ratio_binary_payload(size: usize, seed: u8) -> Vec<u8> {
(0..size)
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
.collect()
}
#[tokio::test]
async fn compressed_multipart_full_get_handles_high_ratio_binary_payload() {
let part_sizes = [5 * 1024 * 1024_usize, 1024 * 1024];
let mut plaintext = Vec::new();
let mut stored = Vec::new();
let mut parts = Vec::with_capacity(part_sizes.len());
for (i, part_size) in part_sizes.iter().enumerate() {
let part_plaintext = high_ratio_binary_payload(*part_size, if i == 0 { 7 } else { 61 });
let (compressed, index) = compressed_part_fixture(&part_plaintext).await;
parts.push(ObjectPartInfo {
number: i + 1,
size: compressed.len(),
actual_size: *part_size as i64,
index,
..Default::default()
});
stored.extend_from_slice(&compressed);
plaintext.extend_from_slice(&part_plaintext);
}
let mut user_defined = HashMap::new();
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()),
);
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
let fixture = CompressedMultipartFixture {
object_info: ObjectInfo {
bucket: "test-bucket".to_string(),
name: "high-ratio-multipart".to_string(),
size: stored.len() as i64,
etag: Some("6bcf86bed8807b8e78f0fc6e0a53079d-2".to_string()),
parts: Arc::new(parts),
user_defined: Arc::new(user_defined),
..Default::default()
},
stored,
plaintext,
};
let read = read_compressed_multipart(&fixture, None, &ObjectOptions::default()).await;
assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size");
assert_eq!(read, fixture.plaintext, "high-ratio multipart payload must survive the roundtrip");
}
/// Full GET over a compressed multipart object must decode across part
/// boundaries: every part is an independent compressed stream (this is also
/// the on-disk shape written by builds before rustfs/rustfs#5169 disabled
/// multipart compression, so this pins legacy-object readability).
#[tokio::test]
async fn compressed_multipart_full_get_decodes_across_part_boundaries() {
let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 2 * 1024 * 1024, 512 * 1024]).await;
let read = read_compressed_multipart(&fixture, None, &ObjectOptions::default()).await;
assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size");
assert_eq!(read, fixture.plaintext, "full GET must reassemble all parts");
}
#[tokio::test]
async fn compressed_multipart_range_get_crosses_part_boundary() {
let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 2 * 1024 * 1024]).await;
let boundary = 3 * 1024 * 1024_i64;
let rs = HTTPRangeSpec {
is_suffix_length: false,
start: boundary - 100_000,
end: boundary + 100_000 - 1,
};
let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await;
let expected = &fixture.plaintext[(boundary - 100_000) as usize..(boundary + 100_000) as usize];
assert_eq!(read, expected, "boundary-crossing range must splice both parts");
}
#[tokio::test]
async fn compressed_multipart_range_get_seeks_into_later_part() {
let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 4 * 1024 * 1024]).await;
// Deep inside part 2 so the plan skips part 1 entirely and (when the
// part carries an index) seeks within part 2.
let start = 3 * 1024 * 1024_i64 + 2 * 1024 * 1024_i64 + 137;
let rs = HTTPRangeSpec {
is_suffix_length: false,
start,
end: start + 64 * 1024 - 1,
};
let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await;
let expected = &fixture.plaintext[start as usize..(start + 64 * 1024) as usize];
assert_eq!(read, expected, "range inside a later part must decode from that part");
}
/// Parts written without a compression index (small parts skip the index in
/// the rio-v2 backend) must still be rangeable: the plan starts at the part
/// boundary and skips decompressed bytes.
#[tokio::test]
async fn compressed_multipart_range_get_works_without_part_indexes() {
let mut fixture = compressed_multipart_fixture(&[1024 * 1024, 1024 * 1024]).await;
let parts = fixture
.object_info
.parts
.iter()
.map(|part| ObjectPartInfo {
index: None,
..part.clone()
})
.collect::<Vec<_>>();
fixture.object_info.parts = Arc::new(parts);
let start = 1024 * 1024_i64 + 4096;
let rs = HTTPRangeSpec {
is_suffix_length: false,
start,
end: start + 32 * 1024 - 1,
};
let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await;
let expected = &fixture.plaintext[start as usize..(start + 32 * 1024) as usize];
assert_eq!(read, expected, "index-less parts must fall back to part-boundary skip");
}
#[tokio::test]
async fn compressed_multipart_part_number_get_returns_single_part() {
let part_sizes = [3 * 1024 * 1024, 2 * 1024 * 1024, 512 * 1024];
let fixture = compressed_multipart_fixture(&part_sizes).await;
let mut logical_offset = 0_usize;
for (i, part_size) in part_sizes.iter().enumerate() {
let opts = ObjectOptions {
part_number: Some(i + 1),
..Default::default()
};
let read = read_compressed_multipart(&fixture, None, &opts).await;
let expected = &fixture.plaintext[logical_offset..logical_offset + part_size];
assert_eq!(read.len(), *part_size, "partNumber={} GET must return the part's logical size", i + 1);
assert_eq!(read, expected, "partNumber={} GET must return the original part bytes", i + 1);
logical_offset += part_size;
}
}
#[tokio::test]
async fn compressed_multipart_suffix_range_reads_tail() {
let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 1024 * 1024]).await;
let suffix_len = 128 * 1024_i64;
let rs = HTTPRangeSpec {
is_suffix_length: true,
start: suffix_len,
end: -1,
};
let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await;
let expected = &fixture.plaintext[fixture.plaintext.len() - suffix_len as usize..];
assert_eq!(read, expected, "suffix range must return the tail of the last part");
}
/// Builds an SSE-C + disk-compression multipart object exactly like the
/// write path: each part is compressed into its own stream and then
/// encrypted with the per-part key schedule. The fixture is
/// legacy-encryption-specific (`rustfs_rio::EncryptReader`), matching the
/// pre-existing `build_legacy_ssec_multipart_fixture` shape, while the
/// compression layer follows the active backend feature.
async fn compressed_encrypted_multipart_fixture(key_bytes: [u8; 32], part_sizes: &[usize]) -> CompressedMultipartFixture {
let pattern = b"compressed encrypted multipart fixture data ";
let mut plaintext = Vec::new();
let mut stored = Vec::new();
let mut parts = Vec::with_capacity(part_sizes.len());
for (i, part_size) in part_sizes.iter().enumerate() {
let part_number = i + 1;
let mut part_plaintext = Vec::with_capacity(*part_size);
while part_plaintext.len() < *part_size {
part_plaintext.extend_from_slice(pattern);
part_plaintext.push(part_number as u8);
}
part_plaintext.truncate(*part_size);
let (compressed, index) = compressed_part_fixture(&part_plaintext).await;
let mut part_cipher = Vec::new();
rustfs_rio::EncryptReader::new_multipart(Cursor::new(compressed), key_bytes, LEGACY_FIXTURE_BASE_NONCE, part_number)
.read_to_end(&mut part_cipher)
.await
.expect("encrypt compressed fixture part");
parts.push(ObjectPartInfo {
number: part_number,
size: part_cipher.len(),
actual_size: *part_size as i64,
index,
..Default::default()
});
stored.extend_from_slice(&part_cipher);
plaintext.extend_from_slice(&part_plaintext);
}
let mut user_defined = legacy_ssec_multipart_metadata(key_bytes, plaintext.len());
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()),
);
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
let object_info = ObjectInfo {
bucket: "test-bucket".to_string(),
name: "compressed-encrypted-multipart".to_string(),
size: stored.len() as i64,
etag: Some(format!("6bcf86bed8807b8e78f0fc6e0a53079d-{}", part_sizes.len())),
parts: Arc::new(parts),
user_defined: Arc::new(user_defined),
..Default::default()
};
CompressedMultipartFixture {
object_info,
stored,
plaintext,
}
}
async fn read_compressed_encrypted_multipart(
fixture: &CompressedMultipartFixture,
key_bytes: [u8; 32],
rs: Option<HTTPRangeSpec>,
opts: &ObjectOptions,
) -> Vec<u8> {
let headers = ssec_headers_from_key(key_bytes);
let (_, offset, length) =
GetObjectReader::new(Box::new(Cursor::new(Vec::new())), rs.clone(), &fixture.object_info, opts, &headers)
.await
.expect("plan compressed encrypted multipart read");
let end = offset + usize::try_from(length).expect("storage window length must be non-negative");
assert!(
end <= fixture.stored.len(),
"planned storage window {offset}..{end} exceeds stored stream of {} bytes",
fixture.stored.len()
);
let window = fixture.stored[offset..end].to_vec();
let (mut reader, replay_offset, replay_length) =
GetObjectReader::new(Box::new(Cursor::new(window)), rs, &fixture.object_info, opts, &headers)
.await
.expect("build compressed encrypted multipart reader");
assert_eq!((replay_offset, replay_length), (offset, length), "read plan must be deterministic");
reader.read_all().await.expect("read compressed encrypted multipart stream")
}
#[tokio::test]
async fn compressed_encrypted_multipart_full_get_roundtrip() {
let key_bytes = [0x6Eu8; 32];
let fixture = compressed_encrypted_multipart_fixture(key_bytes, &[3 * 1024 * 1024, 1024 * 1024]).await;
let read = read_compressed_encrypted_multipart(&fixture, key_bytes, None, &ObjectOptions::default()).await;
assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size");
assert_eq!(read, fixture.plaintext, "SSE-C + compression full GET must reassemble all parts");
}
#[tokio::test]
async fn compressed_encrypted_multipart_range_crosses_part_boundary() {
let key_bytes = [0x6Eu8; 32];
let fixture = compressed_encrypted_multipart_fixture(key_bytes, &[3 * 1024 * 1024, 1024 * 1024]).await;
let boundary = 3 * 1024 * 1024_i64;
let rs = HTTPRangeSpec {
is_suffix_length: false,
start: boundary - 65_536,
end: boundary + 65_536 - 1,
};
let read = read_compressed_encrypted_multipart(&fixture, key_bytes, Some(rs), &ObjectOptions::default()).await;
let expected = &fixture.plaintext[(boundary - 65_536) as usize..(boundary + 65_536) as usize];
assert_eq!(read, expected, "SSE-C + compression boundary-crossing range must splice both parts");
}
#[tokio::test]
async fn compressed_encrypted_multipart_part_number_get_returns_single_part() {
let key_bytes = [0x6Eu8; 32];
let part_sizes = [3 * 1024 * 1024, 1024 * 1024];
let fixture = compressed_encrypted_multipart_fixture(key_bytes, &part_sizes).await;
let opts = ObjectOptions {
part_number: Some(2),
..Default::default()
};
let read = read_compressed_encrypted_multipart(&fixture, key_bytes, None, &opts).await;
let expected = &fixture.plaintext[part_sizes[0]..];
assert_eq!(read.len(), part_sizes[1], "partNumber=2 GET must return the part's logical size");
assert_eq!(read, expected, "partNumber=2 GET must return the original part bytes");
}
#[tokio::test] #[tokio::test]
async fn test_get_object_reader_rejects_ssec_read_without_headers() { async fn test_get_object_reader_rejects_ssec_read_without_headers() {
let object_info = ObjectInfo { let object_info = ObjectInfo {
@@ -23,10 +23,6 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use tokio::task::JoinSet; use tokio::task::JoinSet;
#[allow(
dead_code,
reason = "default operation label for the test-only AsyncBatchProcessor::new (backlog#1823)"
)]
const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom"; const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
const BATCH_PROCESSOR_OPERATION_READ: &str = "read"; const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write"; const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
@@ -215,7 +211,6 @@ pub struct AsyncBatchProcessor {
} }
impl AsyncBatchProcessor { impl AsyncBatchProcessor {
#[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")]
pub fn new(max_concurrent: usize) -> Self { pub fn new(max_concurrent: usize) -> Self {
Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM) Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
} }
@@ -26,26 +26,11 @@ use std::sync::atomic::Ordering;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tracing::warn; use tracing::warn;
/// Dead ecstore-side notification skeleton.
///
/// The working notification stack is `rustfs-notify`, whose own `EventNotifier`
/// is the one bucket configuration actually drives. Nothing calls the methods
/// below; `init_bucket_targets` even logs that it is a no-op in this build.
/// Removing it means also retiring the `InstanceContext` slot that holds it
/// (backlog#939 Phase 5), so it is left explicit here rather than half-removed.
#[allow(
dead_code,
reason = "ecstore-side notification skeleton superseded by rustfs-notify; see module note (backlog#1823)"
)]
pub struct EventNotifier { pub struct EventNotifier {
target_list: TargetList, target_list: TargetList,
//bucket_rules_map: HashMap<String , HashMap<EventName, Rules>>, //bucket_rules_map: HashMap<String , HashMap<EventName, Rules>>,
} }
#[allow(
dead_code,
reason = "ecstore-side notification skeleton superseded by rustfs-notify; see module note (backlog#1823)"
)]
impl EventNotifier { impl EventNotifier {
pub fn new() -> Arc<RwLock<Self>> { pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self { Arc::new(RwLock::new(Self {
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
// #730: background service owners still contain staged notification/rebalance/tier paths. // #730: background service owners still contain staged notification/rebalance/tier paths.
#![allow(dead_code)]
pub(crate) mod batch_processor; pub(crate) mod batch_processor;
pub(crate) mod event_notification; pub(crate) mod event_notification;
+70 -153
View File
@@ -44,14 +44,12 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe";
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100); const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5); const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
/// Cached result from the last successful admin call to a peer. /// Cached result from the last successful admin call to a peer.
struct PeerAdminCache { struct PeerAdminCache {
@@ -97,15 +95,15 @@ lazy_static! {
} }
#[derive(Clone)] #[derive(Clone)]
struct FleetCapabilityProof { struct RemoteVersionStateFleetProof {
topology_fingerprint: String, topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>, peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant, expires_at: Instant,
} }
impl FleetCapabilityProof { impl RemoteVersionStateFleetProof {
fn token(&self) -> FleetCapabilityProofToken { fn token(&self) -> RemoteVersionStateFleetProofToken {
FleetCapabilityProofToken { RemoteVersionStateFleetProofToken {
topology_fingerprint: self.topology_fingerprint.clone(), topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(), peer_epochs: self.peer_epochs.clone(),
} }
@@ -113,41 +111,37 @@ impl FleetCapabilityProof {
} }
#[derive(Clone, PartialEq, Eq)] #[derive(Clone, PartialEq, Eq)]
struct FleetCapabilityProofToken { pub(crate) struct RemoteVersionStateFleetProofToken {
topology_fingerprint: String, topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>, peer_epochs: Arc<BTreeMap<String, Uuid>>,
} }
#[derive(Default)] #[derive(Default)]
struct FleetCapabilityProofState { struct RemoteVersionStateFleetProofState {
proof: Option<FleetCapabilityProof>, proof: Option<RemoteVersionStateFleetProof>,
topology_conflict: bool, topology_conflict: bool,
} }
#[derive(Clone, PartialEq, Eq)] static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
#[derive(Clone, PartialEq, Eq)]
pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> { fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
} }
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> { fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
} }
fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) { fn replace_remote_version_state_fleet_proof_in(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
proof: Option<RemoteVersionStateFleetProof>,
) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof; slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
} }
fn publish_fleet_capability_probe_result( fn publish_remote_version_state_probe_result(
slot: &std::sync::RwLock<FleetCapabilityProofState>, slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
topology_fingerprint: &str, topology_fingerprint: &str,
result: Result<BTreeMap<String, Uuid>>, result: Result<BTreeMap<String, Uuid>>,
observed_at: Instant, observed_at: Instant,
@@ -161,7 +155,7 @@ fn publish_fleet_capability_probe_result(
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs) .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs)) .map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs)); .unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(FleetCapabilityProof { state.proof = Some(RemoteVersionStateFleetProof {
topology_fingerprint: topology_fingerprint.to_string(), topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs, peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL, expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
@@ -169,7 +163,7 @@ fn publish_fleet_capability_probe_result(
None None
} }
Err(err) => { Err(err) => {
replace_fleet_capability_proof(slot, None); replace_remote_version_state_fleet_proof_in(slot, None);
Some(err) Some(err)
} }
} }
@@ -180,60 +174,27 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersion
let state = remote_version_state_fleet_proof_slot() let state = remote_version_state_fleet_proof_slot()
.read() .read()
.unwrap_or_else(std::sync::PoisonError::into_inner); .unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(RemoteVersionStateFleetProofToken) acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
} }
fn acquire_fleet_capability_proof_from( fn acquire_remote_version_state_fleet_proof_from(
state: &FleetCapabilityProofState, state: &RemoteVersionStateFleetProofState,
expected_topology: &str, expected_topology: &str,
now: Instant, now: Instant,
) -> Option<FleetCapabilityProofToken> { ) -> Option<RemoteVersionStateFleetProofToken> {
if state.topology_conflict || !fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, now) { if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
return None; return None;
} }
state.proof.as_ref().map(FleetCapabilityProof::token) state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
} }
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool { pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0)
}
pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = cross_pool_fence_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken)
}
pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool {
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(any(test, feature = "test-util"))]
pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(current) = state.proof.as_ref() else {
return false;
};
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: current.topology_fingerprint.clone(),
peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
expires_at: current.expires_at,
});
true
}
fn fleet_capability_proof_matches(
slot: &std::sync::RwLock<FleetCapabilityProofState>,
proof: &FleetCapabilityProofToken,
) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false; return false;
}; };
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner); let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict { if state.topology_conflict {
return false; return false;
} }
@@ -245,17 +206,13 @@ fn fleet_capability_proof_matches(
}) })
} }
fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
}
#[cfg(test)] #[cfg(test)]
pub(crate) struct RemoteVersionStateFleetProofGuard; pub(crate) struct RemoteVersionStateFleetProofGuard;
#[cfg(test)] #[cfg(test)]
impl Drop for RemoteVersionStateFleetProofGuard { impl Drop for RemoteVersionStateFleetProofGuard {
fn drop(&mut self) { fn drop(&mut self) {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); replace_remote_version_state_fleet_proof(None);
} }
} }
@@ -270,7 +227,7 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
Err(_) => panic!("remote version state test topology is already bound to another fingerprint"), Err(_) => panic!("remote version state test topology is already bound to another fingerprint"),
} }
let peer_epochs = BTreeMap::new(); let peer_epochs = BTreeMap::new();
if let Some(err) = publish_fleet_capability_probe_result( if let Some(err) = publish_remote_version_state_probe_result(
remote_version_state_fleet_proof_slot(), remote_version_state_fleet_proof_slot(),
topology_fingerprint, topology_fingerprint,
Ok(peer_epochs), Ok(peer_epochs),
@@ -281,6 +238,14 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
RemoteVersionStateFleetProofGuard RemoteVersionStateFleetProofGuard
} }
fn remote_version_state_fleet_proof_valid_at(
proof: Option<&RemoteVersionStateFleetProof>,
expected_topology: &str,
now: Instant,
) -> bool {
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
}
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> { fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() { if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
return Err(Error::other("remote version state capability peer identity is invalid")); return Err(Error::other("remote version state capability peer identity is invalid"));
@@ -291,11 +256,11 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] { let mut state = remote_version_state_fleet_proof_slot()
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); .write()
state.topology_conflict = true; .unwrap_or_else(std::sync::PoisonError::into_inner);
state.proof = None; state.topology_conflict = true;
} state.proof = None;
} }
return; return;
} }
@@ -316,23 +281,13 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
} }
None => Err(Error::other("remote version state fleet capability notification system is unavailable")), None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
}; };
let fence_result = match get_global_notification_sys() {
Some(notification_sys) => timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
)
.await
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
};
let topology_conflict = remote_version_state_fleet_proof_slot() let topology_conflict = remote_version_state_fleet_proof_slot()
.read() .read()
.unwrap_or_else(std::sync::PoisonError::into_inner) .unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict; .topology_conflict;
if topology_conflict { if topology_conflict {
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); replace_remote_version_state_fleet_proof(None);
replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None); } else if let Some(err) = publish_remote_version_state_probe_result(
} else if let Some(err) = publish_fleet_capability_probe_result(
remote_version_state_fleet_proof_slot(), remote_version_state_fleet_proof_slot(),
&topology_fingerprint, &topology_fingerprint,
result, result,
@@ -340,24 +295,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
) { ) {
debug!(error = %err, "remote version state fleet capability probe failed closed"); debug!(error = %err, "remote version state fleet capability probe failed closed");
} }
if !topology_conflict
&& let Some(err) = publish_fleet_capability_probe_result(
cross_pool_fence_fleet_proof_slot(),
&topology_fingerprint,
fence_result,
Instant::now(),
)
{
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "cross_pool_fence_v1",
state = "failed_closed",
error = %err,
"notification capability probe"
);
}
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await; sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
} }
}); });
@@ -425,27 +362,6 @@ impl NotificationSys {
} }
Ok(peer_epochs) Ok(peer_epochs)
} }
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
}
let probes = self.peer_clients.iter().map(|client| async {
let client = client
.as_ref()
.ok_or_else(|| Error::other("cross-pool fence capability peer is unreachable"))?;
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
for result in join_all(probes).await {
let (peer, version, epoch) = result?;
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
return Err(Error::other("cross-pool fence capability version is unsupported"));
}
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
}
} }
pub struct NotificationPeerErr { pub struct NotificationPeerErr {
@@ -1623,7 +1539,6 @@ impl NotificationSys {
workers.peers.remove(host); workers.peers.remove(host);
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn tier_config_reload_worker_active(&self, host: &str) -> bool { fn tier_config_reload_worker_active(&self, host: &str) -> bool {
self.tier_config_reload_workers self.tier_config_reload_workers
.lock() .lock()
@@ -1797,7 +1712,6 @@ where
.map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))? .map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))?
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn call_peer_with_timeout<F, Fut>( async fn call_peer_with_timeout<F, Fut>(
timeout_dur: Duration, timeout_dur: Duration,
host_label: &str, host_label: &str,
@@ -2263,16 +2177,16 @@ mod tests {
let now = Instant::now(); let now = Instant::now();
let mut peer_epochs = BTreeMap::new(); let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4()); peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = FleetCapabilityProof { let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(), topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs), peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1), expires_at: now + Duration::from_secs(1),
}; };
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now)); assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
assert!(!fleet_capability_proof_valid_at(None, "topology-a", now)); assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
} }
#[test] #[test]
@@ -2286,25 +2200,25 @@ mod tests {
#[test] #[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() { fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now(); let now = Instant::now();
let proof = FleetCapabilityProof { let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(), topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()), peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1), expires_at: now + Duration::from_secs(1),
}; };
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
} }
#[test] #[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() { fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now(); let now = Instant::now();
let proof = FleetCapabilityProof { let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(), topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1), expires_at: now + Duration::from_secs(1),
}; };
let captured = proof.token(); let captured = proof.token();
let restarted = FleetCapabilityProof { let restarted = RemoteVersionStateFleetProof {
topology_fingerprint: proof.topology_fingerprint.clone(), topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at, expires_at: proof.expires_at,
@@ -2315,11 +2229,11 @@ mod tests {
#[test] #[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() { fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now(); let now = Instant::now();
let epoch = Uuid::new_v4(); let epoch = Uuid::new_v4();
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]); let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
let original = slot let original = slot
.read() .read()
.expect("proof slot should not poison") .expect("proof slot should not poison")
@@ -2328,7 +2242,9 @@ mod tests {
.expect("successful probe should publish proof") .expect("successful probe should publish proof")
.token(); .token();
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()); assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
);
let renewed = slot let renewed = slot
.read() .read()
.expect("proof slot should not poison") .expect("proof slot should not poison")
@@ -2340,7 +2256,8 @@ mod tests {
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!( assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)).is_none() publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
.is_none()
); );
let replaced = slot let replaced = slot
.read() .read()
@@ -2355,18 +2272,18 @@ mod tests {
#[test] #[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() { fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now(); let now = Instant::now();
let mut state = FleetCapabilityProofState { let mut state = RemoteVersionStateFleetProofState {
proof: Some(FleetCapabilityProof { proof: Some(RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(), topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()), peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1), expires_at: now + Duration::from_secs(1),
}), }),
topology_conflict: false, topology_conflict: false,
}; };
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some()); assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
state.topology_conflict = true; state.topology_conflict = true;
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_none()); assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
} }
#[test] #[test]
@@ -2382,19 +2299,19 @@ mod tests {
#[test] #[test]
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() { fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now(); let now = Instant::now();
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some()); assert!(slot.read().expect("proof slot should not poison").proof.is_some());
assert!( assert!(
publish_fleet_capability_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
); );
assert!(slot.read().expect("proof slot should not poison").proof.is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_none());
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some()); assert!(slot.read().expect("proof slot should not poison").proof.is_some());
} }
@@ -864,10 +864,6 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance
RebalanceMetaMergeOutcome::Merged RebalanceMetaMergeOutcome::Merged
} }
#[allow(
dead_code,
reason = "stop-transition helper retained beside stop_rebalance_meta_snapshot; no caller yet (backlog#1823)"
)]
pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) { pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) {
for pool_stat in meta.pool_stats.iter_mut() { for pool_stat in meta.pool_stats.iter_mut() {
if pool_stat.info.status == RebalStatus::Started { if pool_stat.info.status == RebalStatus::Started {
@@ -968,7 +964,6 @@ pub(super) fn rollback_rebalance_start_meta_snapshot_for_id(
}) })
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> { pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option<RebalanceMeta> {
let meta = meta?; let meta = meta?;
stop_rebalance_state(meta, now); stop_rebalance_state(meta, now);
@@ -171,7 +171,6 @@ where
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, D, DFut, W, WFut>( pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, D, DFut, W, WFut>(
set: &Backend, set: &Backend,
bucket: String, bucket: String,
@@ -1,4 +1,5 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::sync::Arc;
use time::OffsetDateTime; use time::OffsetDateTime;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
@@ -31,6 +32,8 @@ pub struct RebalanceStats {
pub cleanup_warnings: RebalanceCleanupWarnings, pub cleanup_warnings: RebalanceCleanupWarnings,
} }
pub type RStats = Vec<Arc<RebalanceStats>>;
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub(super) struct RebalanceBucketConfigs { pub(super) struct RebalanceBucketConfigs {
pub(super) bucket_incarnation_id: Option<uuid::Uuid>, pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
+1
View File
@@ -30,5 +30,6 @@ pub mod warm_backend_minio;
pub mod warm_backend_r2; pub mod warm_backend_r2;
pub mod warm_backend_rustfs; pub mod warm_backend_rustfs;
pub mod warm_backend_s3; pub mod warm_backend_s3;
pub mod warm_backend_s3sdk;
pub mod warm_backend_tencent; pub mod warm_backend_tencent;
pub mod warm_backend_wasabi; pub mod warm_backend_wasabi;
+20 -10
View File
@@ -488,7 +488,6 @@ impl TierCandidateMutation {
targets targets
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn affected_targets( fn affected_targets(
&self, &self,
manager: &TierConfigMgr, manager: &TierConfigMgr,
@@ -803,7 +802,6 @@ fn tier_persisted_reference_blocks_any_target(
.any(|target| tier_persisted_reference_blocks_target(tier_name, backend_identity, target)) .any(|target| tier_persisted_reference_blocks_target(tier_name, backend_identity, target))
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn tier_object_blocks_target_rebind(object: &ObjectInfo, target: &TierMutationIntentTarget) -> io::Result<bool> { fn tier_object_blocks_target_rebind(object: &ObjectInfo, target: &TierMutationIntentTarget) -> io::Result<bool> {
tier_object_blocks_any_target_rebind(object, std::slice::from_ref(target)) tier_object_blocks_any_target_rebind(object, std::slice::from_ref(target))
} }
@@ -2728,6 +2726,14 @@ impl TierConfigMgr {
Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await
} }
fn begin_publish_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
candidate: &Self,
) -> std::result::Result<TierPublishTransition, AdminError> {
Self::begin_publish_transition_with_allowed_mutation_blocks(handle, manager, candidate, None)
}
fn begin_publish_transition_with_allowed_mutation_blocks( fn begin_publish_transition_with_allowed_mutation_blocks(
handle: &Arc<RwLock<Self>>, handle: &Arc<RwLock<Self>>,
manager: &mut Self, manager: &mut Self,
@@ -2813,6 +2819,14 @@ impl TierConfigMgr {
}) })
} }
async fn publish_candidate_inner(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<&str>,
) -> std::result::Result<(), AdminError> {
Self::publish_candidate_inner_with_allowed_mutation_blocks(handle, candidate, driver_tier, None).await
}
async fn publish_candidate_inner_with_allowed_mutation_blocks( async fn publish_candidate_inner_with_allowed_mutation_blocks(
handle: &Arc<RwLock<Self>>, handle: &Arc<RwLock<Self>>,
candidate: Self, candidate: Self,
@@ -2925,7 +2939,6 @@ impl TierConfigMgr {
admin_err admin_err
} }
#[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
async fn publish_candidate_owned( async fn publish_candidate_owned(
handle: &Arc<RwLock<Self>>, handle: &Arc<RwLock<Self>>,
candidate: Self, candidate: Self,
@@ -3528,7 +3541,6 @@ impl TierConfigMgr {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Remove(tier_name.to_string(), force)).await Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Remove(tier_name.to_string(), force)).await
} }
#[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
async fn remove_and_save_with<S>( async fn remove_and_save_with<S>(
handle: &Arc<RwLock<Self>>, handle: &Arc<RwLock<Self>>,
api: Arc<S>, api: Arc<S>,
@@ -3562,7 +3574,6 @@ impl TierConfigMgr {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Clear(force)).await Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Clear(force)).await
} }
#[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
async fn clear_and_save_with<S>( async fn clear_and_save_with<S>(
handle: &Arc<RwLock<Self>>, handle: &Arc<RwLock<Self>>,
api: Arc<S>, api: Arc<S>,
@@ -3601,10 +3612,6 @@ impl TierConfigMgr {
} }
#[cfg(test)] #[cfg(test)]
#[allow(
dead_code,
reason = "lease accounting asserted by a bucket_lifecycle_ops test behind `--features test-util` (backlog#1823)"
)]
pub(crate) async fn active_operation_lease_count(handle: &Arc<RwLock<Self>>, tier_name: &str) -> usize { pub(crate) async fn active_operation_lease_count(handle: &Arc<RwLock<Self>>, tier_name: &str) -> usize {
let manager = handle.read().await; let manager = handle.read().await;
let Some(runtime) = registered_tier_driver_runtime(&manager) else { let Some(runtime) = registered_tier_driver_runtime(&manager) else {
@@ -3710,6 +3717,10 @@ impl TierConfigMgr {
Ok(()) Ok(())
} }
fn retire_driver(&mut self, tier_name: &str) {
self.revoke_driver(tier_name);
}
fn revoke_all_drivers(&mut self) { fn revoke_all_drivers(&mut self) {
if let Some(runtime) = registered_tier_driver_runtime(self) { if let Some(runtime) = registered_tier_driver_runtime(self) {
let mut runtime = lock_unpoisoned(&runtime); let mut runtime = lock_unpoisoned(&runtime);
@@ -3873,7 +3884,6 @@ impl TierConfigMgr {
self.save_config(api, &config_file, data).await self.save_config(api, &config_file, data).await
} }
#[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
async fn save_tiering_config_if_current<S>( async fn save_tiering_config_if_current<S>(
&self, &self,
api: Arc<S>, api: Arc<S>,
@@ -305,10 +305,6 @@ impl TierMutationIntent {
} }
} }
#[allow(
dead_code,
reason = "intent-record persistence asserted by store::init tests (backlog#1823)"
)]
pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result<String> { pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result<String> {
tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id) tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id)
} }
@@ -321,10 +317,6 @@ fn tier_mutation_intent_record_object_name_with_prefix(prefix: &str, mutation_id
Ok(format!("{}/{}/{}/{}.json", prefix, &mutation_key[..2], &mutation_key[2..4], mutation_key)) Ok(format!("{}/{}/{}/{}.json", prefix, &mutation_key[..2], &mutation_key[2..4], mutation_key))
} }
#[allow(
dead_code,
reason = "intent-record persistence asserted by store::init tests (backlog#1823)"
)]
pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result<Uuid> { pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result<Uuid> {
tier_mutation_intent_id_from_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, object) tier_mutation_intent_id_from_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, object)
} }
@@ -363,10 +355,6 @@ fn tier_mutation_intent_id_from_record_object_name_with_prefix(prefix: &str, obj
Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid")) Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid"))
} }
#[allow(
dead_code,
reason = "intent-record persistence asserted by store::init tests (backlog#1823)"
)]
pub(crate) async fn save_tier_mutation_intent_record<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()> pub(crate) async fn save_tier_mutation_intent_record<S>(api: Arc<S>, intent: &TierMutationIntent) -> EcstoreResult<()>
where where
S: EcstoreObjectIO, S: EcstoreObjectIO,
@@ -458,10 +446,6 @@ where
Ok((intent, etag)) Ok((intent, etag))
} }
#[allow(
dead_code,
reason = "intent-record persistence asserted by store::init tests (backlog#1823)"
)]
pub(crate) async fn save_tier_mutation_intent_record_if_current<S>( pub(crate) async fn save_tier_mutation_intent_record_if_current<S>(
api: Arc<S>, api: Arc<S>,
intent: &TierMutationIntent, intent: &TierMutationIntent,
@@ -41,7 +41,10 @@ use crate::services::tier::{
}; };
use tracing::warn; use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> { fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
if remote_version.is_empty() { if remote_version.is_empty() {
@@ -61,6 +64,7 @@ pub struct WarmBackendGCS {
pub control: Arc<StorageControl>, pub control: Arc<StorageControl>,
pub bucket: String, pub bucket: String,
pub prefix: String, pub prefix: String,
pub storage_class: String,
} }
impl WarmBackendGCS { impl WarmBackendGCS {
@@ -100,6 +104,7 @@ impl WarmBackendGCS {
control, control,
bucket: conf.bucket.clone(), bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}) })
} }
@@ -33,6 +33,8 @@ use crate::client::{
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}, transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl}, transition_api::{ReadCloser, ReaderImpl},
}; };
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::services::tier::{ use crate::services::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{ warm_backend::{
@@ -0,0 +1,200 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use crate::client::{
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
pub struct WarmBackendS3 {
pub client: Arc<Client>,
pub bucket: String,
pub prefix: String,
pub storage_class: String,
}
impl WarmBackendS3 {
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
let u = match Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
{
return Err(std::io::Error::other("both the token file and the role ARN are required"));
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required"));
} else if conf.aws_role
&& (conf.aws_role_web_identity_token_file != ""
|| conf.aws_role_arn != ""
|| conf.access_key != ""
|| conf.secret_key != "")
{
return Err(std::io::Error::other(
"AWS Role cannot be activated with static credentials or the web identity token file",
));
} else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let creds;
if conf.access_key != "" && conf.secret_key != "" {
creds = Credentials::new(
conf.access_key.clone(), // access_key_id
conf.secret_key.clone(), // secret_access_key
None, // session_token (optional)
None,
"Static",
);
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let region_provider = RegionProviderChain::default_provider().or_else(Region::new(conf.region.clone()));
#[allow(deprecated)]
let config = aws_config::from_env()
.endpoint_url(conf.endpoint.clone())
.region(region_provider)
.credentials_provider(creds)
.load()
.await;
let client = Client::new(&config);
let client = Arc::new(client);
Ok(Self {
client,
bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(),
storage_class: conf.storage_class.clone(),
})
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
}
return dest_obj;
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.put_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.body(match r {
ReaderImpl::Body(content_body) => ByteStream::from(content_body.to_vec()),
ReaderImpl::ObjectBody(mut content_body) => ByteStream::from(content_body.read_all().await?),
})
.send()
.await
else {
return Err(std::io::Error::other("put_object error"));
};
Ok(res.version_id().unwrap_or("").to_string())
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let client = self.client.clone();
let mut req = client.get_object().bucket(&self.bucket).key(&self.get_dest(object));
if !rv.is_empty() {
req = req.version_id(rv);
}
if opts.start_offset >= 0 && opts.length > 0 {
let end = opts
.start_offset
.checked_add(opts.length)
.and_then(|v| v.checked_sub(1))
.ok_or_else(|| std::io::Error::other("invalid range: overflow"))?;
req = req.range(format!("bytes={}-{}", opts.start_offset, end));
}
let res = req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(ReadCloser::new(std::io::Cursor::new(
res.body.collect().await.map(|data| data.into_bytes().to_vec())?,
)))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
let client = self.client.clone();
let mut req = client.delete_object().bucket(&self.bucket).key(&self.get_dest(object));
if !rv.is_empty() {
req = req.version_id(rv);
}
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(())
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.list_objects_v2()
.bucket(&self.bucket)
//.max_keys(10)
//.into_paginator()
.send()
.await
else {
return Err(std::io::Error::other("list_objects_v2 error"));
};
Ok(res.common_prefixes.unwrap_or_default().len() > 0 || res.contents.unwrap_or_default().len() > 0)
}
}
+69 -366
View File
@@ -32,28 +32,20 @@ use crate::diagnostics::get::{
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK,
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
}; };
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{ use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
@@ -598,7 +590,7 @@ impl MetadataQuorumAccumulator {
}) })
} }
pub(crate) fn default_write_quorum(&self) -> usize { pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks; return self.total_disks;
} }
@@ -659,48 +651,36 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
&& left.erasure.distribution == right.erasure.distribution && left.erasure.distribution == right.erasure.distribution
} }
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
bucket: &str, bucket: &str,
object: &str, object: &str,
candidate: &FileInfo, candidate: &FileInfo,
parts_metadata: &[FileInfo], parts_metadata: &[FileInfo],
disks: &[Option<DiskStore>], disks: &[Option<DiskStore>],
) -> Option<&'static str> { ) -> bool {
if !candidate.inline_data() { if !candidate.inline_data()
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); || candidate.is_compressed()
}
if candidate.is_compressed()
|| candidate || candidate
.metadata .metadata
.keys() .keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key)) .any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|| candidate.is_remote()
|| candidate.deleted
|| candidate.size <= 0
|| candidate.parts.len() != 1
|| !candidate.has_valid_erasure_geometry()
{ {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED); return false;
}
if candidate.is_remote() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
}
if candidate.deleted {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
}
if candidate.size <= 0 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}
if candidate.parts.len() != 1 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
}
if !candidate.has_valid_erasure_geometry() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
} }
let Ok(object_size) = usize::try_from(candidate.size) else { let Ok(object_size) = usize::try_from(candidate.size) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); return false;
}; };
if candidate.parts.first().is_none_or(|part| part.size != object_size) { if candidate.parts.first().is_none_or(|part| part.size != object_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); return false;
} }
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); return false;
} }
let Ok(erasure) = coding::Erasure::try_new_with_options( let Ok(erasure) = coding::Erasure::try_new_with_options(
@@ -709,18 +689,18 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
candidate.erasure.block_size, candidate.erasure.block_size,
candidate.uses_legacy_checksum, candidate.uses_legacy_checksum,
) else { ) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); return false;
}; };
let data_files = let Some(data_files) =
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| {
disks.get(index).is_some_and(Option::is_some) disks.get(index).is_some_and(Option::is_some)
}) { })
Ok(data_files) => data_files, else {
Err(reason) => return Some(reason), return false;
}; };
let Some(part) = candidate.parts.first() else { let Some(part) = candidate.parts.first() else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); return false;
}; };
let checksum_info = candidate.erasure.get_checksum_info(part.number); let checksum_info = candidate.erasure.get_checksum_info(part.number);
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
@@ -740,13 +720,12 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
let Ok(mut readers) = let Ok(mut readers) =
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
else { else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); return false;
}; };
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size)
Some(body) if body.len() == object_size => None, .await
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), .is_some_and(|body| body.len() == object_size)
}
} }
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
@@ -2489,7 +2468,6 @@ impl SetDisks {
let mut next_fanout_index = 0usize; let mut next_fanout_index = 0usize;
let mut scheduled_count = 0usize; let mut scheduled_count = 0usize;
let mut force_full_wait = false; let mut force_full_wait = false;
let mut final_miss_reason_override = None;
let spawn_read_version = let spawn_read_version =
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| { |join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
let task_opts = opts; let task_opts = opts;
@@ -2562,29 +2540,17 @@ impl SetDisks {
.or_else(|| accumulator.version_early_stop_decision()) .or_else(|| accumulator.version_early_stop_decision())
{ {
let should_return_early = if read_data { let should_return_early = if read_data {
match accumulator.candidate.as_ref() { let allow_data_read_early_stop = match accumulator.candidate.as_ref() {
Some(candidate) => match data_read_early_stop_inline_body_miss_reason( Some(candidate) => {
bucket.as_ref(), data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks)
object.as_ref(), .await
candidate,
&ress,
disks,
)
.await
{
None => true,
Some(reason) => {
force_full_wait = true;
final_miss_reason_override = Some(reason);
false
}
},
None => {
force_full_wait = true;
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
false
} }
None => false,
};
if !allow_data_read_early_stop {
force_full_wait = true;
} }
allow_data_read_early_stop
} else { } else {
true true
}; };
@@ -2646,12 +2612,7 @@ impl SetDisks {
} }
} }
let accumulator_miss_reason = accumulator.final_miss_reason(); rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason());
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
_ => accumulator_miss_reason,
};
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
@@ -3050,41 +3011,12 @@ impl RenameConvergence {
} }
} }
pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) online_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) convergence: RenameConvergence,
pub(in crate::set_disk) data_dir: Option<Uuid>,
pub(in crate::set_disk) cleanup_disks: Vec<Option<DiskStore>>,
pub(in crate::set_disk) old_current_size: Option<OldCurrentSize>,
pub(in crate::set_disk) committed_file_info: FileInfo,
}
type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>,
RenameConvergence,
Option<Uuid>,
Vec<Option<DiskStore>>,
Option<OldCurrentSize>,
);
impl RenameDataCommit {
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
(
self.online_disks,
self.convergence,
self.data_dir,
self.cleanup_disks,
self.old_current_size,
)
}
}
impl SetDisks { impl SetDisks {
pub(in crate::set_disk) fn default_read_quorum(&self) -> usize { pub(in crate::set_disk) fn default_read_quorum(&self) -> usize {
self.set_drive_count - self.default_parity_count self.set_drive_count - self.default_parity_count
} }
pub(crate) fn default_write_quorum(&self) -> usize { pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
let mut data_count = self.set_drive_count - self.default_parity_count; let mut data_count = self.set_drive_count - self.default_parity_count;
if data_count == self.default_parity_count { if data_count == self.default_parity_count {
data_count += 1 data_count += 1
@@ -3093,97 +3025,6 @@ impl SetDisks {
data_count data_count
} }
pub(in crate::set_disk) async fn prepare_quota_mutation_fences(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
let fence_path = fence_path.clone();
async move {
let disk = disk?;
match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await {
Ok(token) => Some((disk, token)),
Err(_) => None,
}
}
}))
.await;
if results.iter().flatten().count() < write_quorum {
for (disk, token) in results.iter().flatten() {
let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await;
}
return Err(StorageError::ErasureWriteQuorum);
}
let mut fenced_disks = Vec::with_capacity(results.len());
let mut tokens = Vec::with_capacity(results.len());
for result in results {
match result {
Some((disk, token)) => {
fenced_disks.push(Some(disk));
tokens.push(Some(token));
}
None => {
fenced_disks.push(None);
tokens.push(None);
}
}
}
Ok((fenced_disks, tokens))
}
pub(in crate::set_disk) async fn release_quota_mutation_fences(
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<()> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
let disk = disk.as_ref()?.clone();
let token = (*token)?;
let fence_path = fence_path.clone();
Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await })
}))
.await;
if results.iter().filter(|result| result.is_ok()).count() < write_quorum {
return Err(StorageError::ErasureWriteQuorum);
}
Ok(())
}
pub(in crate::set_disk) fn assign_rename_data_indexes(file_infos: &mut [FileInfo]) {
for (index, file_info) in file_infos.iter_mut().enumerate() {
if file_info.erasure.index == 0 {
file_info.erasure.index = index + 1;
}
}
}
pub(in crate::set_disk) async fn abort_quota_reservation_after_fence(
reservation: crate::bucket::quota::reservation::QuotaReservation,
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
fenced: bool,
) {
let safe_to_abort = !fenced
|| Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum)
.await
.is_ok();
if safe_to_abort {
reservation.abort().await;
} else {
reservation.defer_after_fence();
}
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))] #[tracing::instrument(level = "debug", skip(disks, file_infos))]
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
pub(in crate::set_disk) async fn rename_data( pub(in crate::set_disk) async fn rename_data(
@@ -3194,22 +3035,13 @@ impl SetDisks {
dst_bucket: &str, dst_bucket: &str,
dst_object: &str, dst_object: &str,
write_quorum: usize, write_quorum: usize,
) -> disk::error::Result<RenameDataLegacyTuple> { ) -> disk::error::Result<(
Self::rename_data_owned(disks, src_bucket, src_object, file_infos.to_vec(), dst_bucket, dst_object, write_quorum) Vec<Option<DiskStore>>,
.await RenameConvergence,
.map(RenameDataCommit::into_legacy_tuple) Option<Uuid>,
} Vec<Option<DiskStore>>,
Option<OldCurrentSize>,
#[tracing::instrument(level = "debug", skip(disks, file_infos))] )> {
pub(in crate::set_disk) async fn rename_data_owned(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
if let Some(file_info) = disks if let Some(file_info) = disks
.iter() .iter()
.zip(file_infos.iter()) .zip(file_infos.iter())
@@ -3234,7 +3066,7 @@ impl SetDisks {
let disk_count = disks.len(); let disk_count = disks.len();
let fanout_disks = disks.to_vec(); let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos; let fanout_file_infos = file_infos.to_vec();
let fanout_src_bucket = src_bucket.clone(); let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone(); let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone(); let fanout_dst_bucket = dst_bucket.clone();
@@ -3246,9 +3078,9 @@ impl SetDisks {
let fanout = tokio::spawn(async move { let fanout = tokio::spawn(async move {
let futures = fanout_disks let futures = fanout_disks
.into_iter() .into_iter()
.zip(fanout_file_infos.iter()) .zip(fanout_file_infos)
.enumerate() .enumerate()
.map(|(i, (disk, file_info))| { .map(|(i, (disk, mut file_info))| {
let src_bucket = fanout_src_bucket.clone(); let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone(); let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone(); let dst_object = fanout_dst_object.clone();
@@ -3265,15 +3097,11 @@ impl SetDisks {
}; };
let is_delete_marker = file_info.is_canonical_delete_marker(); let is_delete_marker = file_info.is_canonical_delete_marker();
let mut local_file_info; if file_info.erasure.index == 0 {
let file_info = if file_info.erasure.index == 0 { file_info.erasure.index = i + 1;
local_file_info = file_info.clone(); }
local_file_info.erasure.index = i + 1;
&local_file_info if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
} else {
file_info
};
if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) {
return Err(DiskError::FileCorrupt); return Err(DiskError::FileCorrupt);
} }
@@ -3281,13 +3109,12 @@ impl SetDisks {
// A no-op immediately-ready future in production. // A no-op immediately-ready future in production.
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
disk.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await .await
}) })
.catch_unwind() .catch_unwind()
}); });
let results = join_all(futures).await; join_all(futures).await
(results, fanout_file_infos)
}); });
let mut disk_versions = vec![None; disk_count]; let mut disk_versions = vec![None; disk_count];
@@ -3295,7 +3122,7 @@ impl SetDisks {
let mut cleanup_data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count];
let (results, mut file_infos) = fanout.await.map_err(|_| DiskError::Unexpected)?; let results = fanout.await.map_err(|_| DiskError::Unexpected)?;
for (idx, result) in results.iter().enumerate() { for (idx, result) in results.iter().enumerate() {
match result { match result {
@@ -3351,7 +3178,7 @@ impl SetDisks {
} }
if let Some(disk) = disks[i].as_ref() { if let Some(disk) = disks[i].as_ref() {
let fi = std::mem::take(&mut file_infos[i]); let fi = file_infos[i].clone();
let old_data_dir = data_dirs[i]; let old_data_dir = data_dirs[i];
let disk = disk.clone(); let disk = disk.clone();
let dst_bucket = dst_bucket.clone(); let dst_bucket = dst_bucket.clone();
@@ -3474,8 +3301,6 @@ impl SetDisks {
let convergence = Self::classify_rename_convergence(&disk_versions, &errs); let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum); let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
let online_disks = Self::eval_disks(disks, &errs); let online_disks = Self::eval_disks(disks, &errs);
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
let committed_file_info = std::mem::take(&mut file_infos[committed_slot]);
let cleanup_disks = if let Some(data_dir) = data_dir { let cleanup_disks = if let Some(data_dir) = data_dir {
disks disks
.iter() .iter()
@@ -3493,14 +3318,7 @@ impl SetDisks {
vec![None; disks.len()] vec![None; disks.len()]
}; };
Ok(RenameDataCommit { Ok((online_disks, convergence, data_dir, cleanup_disks, old_current_size))
online_disks,
convergence,
data_dir,
cleanup_disks,
old_current_size,
committed_file_info,
})
} }
/// rustfs/backlog#1009: reduce the per-disk observations of the /// rustfs/backlog#1009: reduce the per-disk observations of the
@@ -6105,126 +5923,11 @@ mod tests {
.clone(); .clone();
assert!( assert!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks) data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await,
.await
.is_none(),
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm" "legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
); );
} }
#[tokio::test]
async fn data_read_early_stop_reports_inline_miss_reasons() {
let bucket = "inline-data-get-miss-reason-bucket";
let object = "inline-data-get-miss-reason-object";
let payload = b"verified inline payload";
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
let distribution = files
.first()
.map(|file| file.erasure.distribution.clone())
.expect("fixture should include metadata");
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
let mut parts_metadata = vec![FileInfo::default(); 4];
for disk_index in order.into_iter().take(3) {
let block_index = distribution
.get(disk_index)
.copied()
.expect("fixture distribution should cover every disk");
parts_metadata[disk_index] = files
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
.expect("fixture should include every distributed shard")
.clone();
}
let candidate = parts_metadata
.iter()
.find(|file| file.name == object)
.expect("fixture should include observed metadata")
.clone();
let data_disk = distribution
.iter()
.position(|block_index| *block_index == 1)
.expect("fixture distribution should include first data shard");
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
None
);
let mut not_inline = candidate.clone();
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &not_inline, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
);
let mut transformed = candidate.clone();
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
);
let mut deleted = candidate.clone();
deleted.deleted = true;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
);
let mut zero_size = candidate.clone();
zero_size.size = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
);
let mut multipart = candidate.clone();
multipart.parts.push(multipart.parts[0].clone());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
);
let mut invalid_geometry = candidate.clone();
invalid_geometry.erasure.data_blocks = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
);
let mut missing_shard = parts_metadata.clone();
missing_shard[data_disk] = FileInfo::default();
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
);
let mut missing_payload = parts_metadata.clone();
missing_payload[data_disk].data = None;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
);
let mut identity_mismatch = parts_metadata.clone();
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
);
let mut corrupt = parts_metadata.clone();
if let Some(data) = corrupt[data_disk].data.as_mut() {
let mut corrupt_data = data.to_vec();
corrupt_data[0] ^= 0x01;
*data = Bytes::from(corrupt_data);
}
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
);
}
#[test] #[test]
#[serial_test::serial] #[serial_test::serial]
fn metadata_fanout_lifecycle_records_real_early_stop_abort() { fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
@@ -6314,7 +6017,7 @@ mod tests {
&[ &[
("path", GET_OBJECT_PATH_INTERNAL_META), ("path", GET_OBJECT_PATH_INTERNAL_META),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
], ],
), ),
1, 1,
@@ -6326,7 +6029,7 @@ mod tests {
&[ &[
("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
], ],
), ),
0, 0,
+10 -30
View File
@@ -59,10 +59,7 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::cluster::rpc::heal_bucket_local_on_disks;
use crate::data_usage::record_compression_total_memory; use crate::data_usage::record_compression_total_memory;
use crate::diagnostics::get::{ use crate::diagnostics::get::{
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
@@ -722,8 +719,8 @@ pub(crate) use core::io_primitives::disk_call_counters;
mod ctx; mod ctx;
mod metadata; mod metadata;
mod ops; mod ops;
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len; pub(crate) use ops::object::body_cache_plaintext_len;
@@ -3866,20 +3863,11 @@ fn inline_erasure_shard_file_offset(
} }
fn collect_inline_data_shard_fileinfos_by_index<'a>( fn collect_inline_data_shard_fileinfos_by_index<'a>(
parts_metadata: &'a [FileInfo],
fi: &FileInfo,
data_shards: usize,
disk_is_online: impl FnMut(usize) -> bool,
) -> Option<Vec<&'a FileInfo>> {
collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, fi, data_shards, disk_is_online).ok()
}
fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
parts_metadata: &'a [FileInfo], parts_metadata: &'a [FileInfo],
fi: &FileInfo, fi: &FileInfo,
data_shards: usize, data_shards: usize,
mut disk_is_online: impl FnMut(usize) -> bool, mut disk_is_online: impl FnMut(usize) -> bool,
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> { ) -> Option<Vec<&'a FileInfo>> {
let distribution = &fi.erasure.distribution; let distribution = &fi.erasure.distribution;
let mut data_files = vec![None; data_shards]; let mut data_files = vec![None; data_shards];
@@ -3887,35 +3875,27 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
if !disk_is_online(disk_index) { if !disk_is_online(disk_index) {
continue; continue;
} }
let Some(&block_index) = distribution.get(disk_index) else { let block_index = *distribution.get(disk_index)?;
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
};
if block_index == 0 || block_index > data_shards { if block_index == 0 || block_index > data_shards {
continue; continue;
} }
if file_info.name.is_empty() {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD);
}
if file_info.erasure.index != block_index { if file_info.erasure.index != block_index {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); continue;
} }
if !file_info.has_valid_erasure_geometry() { if !file_info.has_valid_erasure_geometry() {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); continue;
} }
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) { if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); continue;
} }
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) { if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD); continue;
} }
data_files[block_index - 1] = Some(file_info); data_files[block_index - 1] = Some(file_info);
} }
data_files data_files.into_iter().collect()
.into_iter()
.collect::<Option<Vec<_>>>()
.ok_or(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
} }
impl SetDisks { impl SetDisks {
+2 -3
View File
@@ -13,7 +13,6 @@
// limitations under the License. // limitations under the License.
use super::super::*; use super::super::*;
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::io_support::bitrot::object_mmap_read_enabled; use crate::io_support::bitrot::object_mmap_read_enabled;
use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use tracing::trace; use tracing::trace;
@@ -1165,10 +1164,10 @@ impl SetDisks {
let rename_result = if should_fail_heal_rename(bucket, object, index) { let rename_result = if should_fail_heal_rename(bucket, object, index) {
Err(DiskError::Unexpected) Err(DiskError::Unexpected)
} else { } else {
disk.rename_data_borrowed( disk.rename_data(
RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET,
&tmp_id, &tmp_id,
&parts_metadata[index], parts_metadata[index].clone(),
bucket, bucket,
object, object,
) )
+114 -546
View File
@@ -27,12 +27,11 @@ use super::object::{
object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence, object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence,
verify_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
}; };
use crate::bucket::quota::reservation;
use crate::crash_inject::{self, CrashPoint}; use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing; use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream}; use futures::{StreamExt, stream};
use std::future::Future; use std::future::Future;
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration; use std::time::Duration;
use tokio::task::JoinSet; use tokio::task::JoinSet;
@@ -62,21 +61,20 @@ impl StaleMultipartCleanupGuard {
} }
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
#[derive(Clone, Copy, PartialEq, Eq)] #[derive(Clone, Copy, PartialEq, Eq)]
pub enum MultipartCommitPause { pub(crate) enum MultipartCommitPause {
PutPartBeforeLockAcquire, PutPartBeforeLockAcquire,
PutPartBeforeLockLost, PutPartBeforeLockLost,
PutPartAfterRename, PutPartAfterRename,
BeforeLockLost, BeforeLockLost,
BeforeQuotaRename,
BeforeTransactionEpochVerify, BeforeTransactionEpochVerify,
BeforeObjectPublication, BeforeObjectPublication,
AfterObjectPublication, AfterObjectPublication,
AfterRename, AfterRename,
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
struct MultipartCommitBarrierState { struct MultipartCommitBarrierState {
bucket: String, bucket: String,
object: String, object: String,
@@ -87,22 +85,27 @@ struct MultipartCommitBarrierState {
release: tokio::sync::Semaphore, release: tokio::sync::Semaphore,
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
pub struct MultipartCommitBarrier { pub(crate) struct MultipartCommitBarrier {
state: Arc<MultipartCommitBarrierState>, state: Arc<MultipartCommitBarrierState>,
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<MultipartCommitBarrierState>>>> = static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<MultipartCommitBarrierState>>>> =
std::sync::OnceLock::new(); std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
impl MultipartCommitBarrier { impl MultipartCommitBarrier {
pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self { pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
Self::install_for_arrivals(bucket, object, pause, 1) Self::install_for_arrivals(bucket, object, pause, 1)
} }
pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self { pub(crate) fn install_for_arrivals(
bucket: &str,
object: &str,
pause: MultipartCommitPause,
expected_arrivals: usize,
) -> Self {
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival"); assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
let state = Arc::new(MultipartCommitBarrierState { let state = Arc::new(MultipartCommitBarrierState {
bucket: bucket.to_string(), bucket: bucket.to_string(),
@@ -123,7 +126,7 @@ impl MultipartCommitBarrier {
Self { state } Self { state }
} }
pub async fn wait_until_paused(&self) { pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), async { tokio::time::timeout(Duration::from_secs(30), async {
loop { loop {
let arrived = self.state.arrived.notified(); let arrived = self.state.arrived.notified();
@@ -137,12 +140,12 @@ impl MultipartCommitBarrier {
.expect("multipart completion should reach the deterministic commit barrier"); .expect("multipart completion should reach the deterministic commit barrier");
} }
pub fn release(&self) { pub(crate) fn release(&self) {
self.state.release.add_permits(self.state.expected_arrivals); self.state.release.add_permits(self.state.expected_arrivals);
} }
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
impl Drop for MultipartCommitBarrier { impl Drop for MultipartCommitBarrier {
fn drop(&mut self) { fn drop(&mut self) {
self.release(); self.release();
@@ -156,7 +159,7 @@ impl Drop for MultipartCommitBarrier {
} }
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(test)]
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
let barrier = { let barrier = {
let mut slot = MULTIPART_COMMIT_BARRIER let mut slot = MULTIPART_COMMIT_BARRIER
@@ -1889,41 +1892,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.parts = Vec::with_capacity(uploaded_parts.len()); fi.parts = Vec::with_capacity(uploaded_parts.len());
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let preserve_replication_ciphertext = opts.replication_request
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
if quota_context.is_enforced() && preserve_replication_ciphertext {
return Err(Error::PartMissingOrCorrupt);
}
let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
let data_movement_actual_size = if opts.data_movement {
rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE)
.map(|value| {
value
.parse::<i64>()
.ok()
.filter(|value| *value >= 0)
.ok_or(Error::PartMissingOrCorrupt)
})
.transpose()?
} else {
None
};
let data_movement_actual_size_u64 = data_movement_actual_size
.map(u64::try_from)
.transpose()
.map_err(|_| Error::PartMissingOrCorrupt)?;
let mut object_size: usize = 0; let mut object_size: usize = 0;
let mut object_actual_size: i64 = 0; let mut object_actual_size: i64 = 0;
@@ -2050,29 +2018,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
checksum_combined.extend_from_slice(cs.raw.as_slice()); checksum_combined.extend_from_slice(cs.raw.as_slice());
} }
object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; object_size += ext_part.size;
let unknown_actual_size_allowed = opts.data_movement && transformed_object && data_movement_actual_size.is_some() if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
|| opts.replication_request && !quota_context.is_enforced();
if ext_part.actual_size < 0 && !unknown_actual_size_allowed {
return Err(Error::PartMissingOrCorrupt); return Err(Error::PartMissingOrCorrupt);
} }
let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object {
ext_part
.actual_size
.max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?)
} else {
ext_part.actual_size
};
object_actual_size = object_actual_size object_actual_size = object_actual_size
.checked_add(normalized_actual_size) .checked_add(ext_part.actual_size)
.ok_or(Error::PartMissingOrCorrupt)?; .ok_or(Error::PartMissingOrCorrupt)?;
let mut completed_part = completed_multipart_object_part(p.part_num, ext_part);
completed_part.actual_size = normalized_actual_size; fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
fi.parts.push(completed_part);
}
if !transformed_object && data_movement_actual_size.is_some_and(|actual_size| actual_size < object_actual_size) {
return Err(Error::PartMissingOrCorrupt);
} }
if let Some(wtcs) = opts.want_checksum.as_ref() { if let Some(wtcs) = opts.want_checksum.as_ref() {
if checksum_type.full_object_requested() { if checksum_type.full_object_requested() {
if wtcs.encoded != checksum.encoded { if wtcs.encoded != checksum.encoded {
@@ -2097,35 +2053,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
} }
} }
let declared_replication_actual_size = opts if let Some(admission) = opts.quota_admission {
.replication_request let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
.then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP)) if quota_operation_size > admission.remaining() {
.flatten(); return Err(Error::QuotaExceeded {
let replication_actual_size = if opts.replication_request && quota_context.is_enforced() { current: admission.current_usage(),
let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; limit: admission.quota_limit(),
let declared_cap = declared_replication_actual_size });
.as_deref() }
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt)) }
.transpose()?
.unwrap_or(0);
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata)
.map_err(Error::other)?
.map(u64::try_from)
.transpose()
.map_err(|_| Error::PartMissingOrCorrupt)?
.unwrap_or(0);
Some(observed_size.max(declared_cap).max(declared_encryption_size))
} else {
None
};
let data_movement_quota_size = data_movement_actual_size_u64.filter(|_| quota_context.is_enforced());
let quota_new_size = match data_movement_quota_size.or(replication_actual_size) {
Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
None if quota_context.is_enforced() => u64::try_from(object_actual_size)
.map_err(|_| Error::PartMissingOrCorrupt)?
.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
None => 0,
};
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) { if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) { if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
fi.checksum = Some(Bytes::from(rc_crc_bytes)); fi.checksum = Some(Bytes::from(rc_crc_bytes));
@@ -2177,6 +2113,20 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
); );
} }
let data_movement_actual_size = if opts.data_movement {
rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE)
.map(|value| {
value
.parse::<i64>()
.ok()
.filter(|value| *value >= 0)
.ok_or_else(|| Error::other("data movement actual size metadata is invalid"))
})
.transpose()?
} else {
None
};
if let Some(actual_size) = data_movement_actual_size { if let Some(actual_size) = data_movement_actual_size {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
if persist_encryption_original_size { if persist_encryption_original_size {
@@ -2184,13 +2134,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
} }
} else if opts.replication_request { } else if opts.replication_request {
if let Some(actual_size) = replication_actual_size { if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
if persist_encryption_original_size {
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
}
} else if let Some(actual_size) = declared_replication_actual_size {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
if persist_encryption_original_size { if persist_encryption_original_size {
fi.metadata fi.metadata
@@ -2380,41 +2324,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
} else { } else {
None None
}; };
let quota_old_size = if quota_context.is_enforced() {
if opts.data_movement {
quota_new_size
} else {
reservation::replaced_logical_size(&self, bucket, object, opts).await?
}
} else {
0
};
let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
Ok((disks, tokens)) => {
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
if let Some(token) = token {
insert_str(
&mut metadata.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
token.as_uuid().to_string(),
);
}
}
(disks, tokens)
}
Err(err) => {
quota_reservation.abort().await;
return Err(err);
}
}
} else {
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
};
let transaction_epoch = let transaction_epoch =
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
let commit_set = self.clone(); let commit_set = self.clone();
let commit_bucket = bucket.to_owned(); let commit_bucket = bucket.to_owned();
@@ -2422,152 +2333,49 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let commit_upload_id = upload_id.to_owned(); let commit_upload_id = upload_id.to_owned();
let commit_upload_id_path = upload_id_path.clone(); let commit_upload_id_path = upload_id_path.clone();
let commit_version_suspended = opts.version_suspended; let commit_version_suspended = opts.version_suspended;
let commit_versioned = opts.versioned;
let commit_version_id = opts.version_id.clone();
let commit_namespace_lock_fence = opts.namespace_lock_fence.clone();
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_is_versioned = opts.versioned || opts.version_suspended; let commit_is_versioned = opts.versioned || opts.version_suspended;
let commit_capacity_scope_token = opts.capacity_scope_token; let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_object_lock_guard = object_lock_guard.take(); let commit_object_lock_guard = object_lock_guard.take();
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence; let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some();
let commit = async move { let commit = async move {
let _object_lock_guard = commit_object_lock_guard; let _object_lock_guard = commit_object_lock_guard;
let _upload_guard = upload_guard; let _upload_guard = upload_guard;
let mut quota_reservation = quota_reservation;
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let pre_rename_result: Result<()> = async { // Crash-consistency injection: hard power loss after the upload is fully
// Crash-consistency injection: hard power loss after the upload is fully // staged and locked but before the authoritative rename_data commit. No
// staged and locked but before the authoritative rename_data commit. No // disk has moved the staged data, so a crash here must leave any prior
// disk has moved the staged data, so a crash here must leave any prior // committed version byte-for-byte intact (rustfs/backlog#864) and the
// committed version byte-for-byte intact (rustfs/backlog#864) and the // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { return Err(StorageError::Unexpected);
return Err(StorageError::Unexpected);
}
quota_reservation.mark_commit_started().await?;
#[cfg(any(test, feature = "test-util"))]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await;
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
let restore_opts = ObjectOptions {
version_id: commit_version_id.clone(),
versioned: commit_versioned,
version_suspended: commit_version_suspended,
no_lock: true,
..Default::default()
};
commit_set
.require_current_restore_operation_id(
&commit_bucket,
&commit_object,
&restore_opts,
expected_restore_operation_id,
"complete_multipart_upload_quota_reservation",
)
.await?;
if let Some(proof) = transaction_fencing_proof.as_ref()
&& !object_transaction_fencing_fleet_proof_matches(proof)
{
return Err(Error::other(
"object transaction fencing fleet capability changed during complete_multipart_upload",
));
}
if let Some(expected) = transaction_epoch_fence {
#[cfg(test)]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify)
.await;
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
}
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
Ok(())
}
.await;
if let Err(err) = pre_rename_result {
SetDisks::abort_quota_reservation_after_fence(
quota_reservation,
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
quota_mutation_fence,
)
.await;
return Err(err);
} }
// The trailing `_` drops the rename_data old-size backfill // The trailing `_` drops the rename_data old-size backfill
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
// `get_object_info` lookup, so the backfill has no consumer here yet. // `get_object_info` lookup, so the backfill has no consumer here yet.
Self::assign_rename_data_indexes(&mut parts_metadatas); if let Some(proof) = transaction_fencing_proof.as_ref()
let rename_result = SetDisks::rename_data_owned( && !object_transaction_fencing_fleet_proof_matches(proof)
&commit_disks, {
return Err(Error::other(
"object transaction fencing fleet capability changed during complete_multipart_upload",
));
}
if let Some(expected) = transaction_epoch_fence {
#[cfg(test)]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await;
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
}
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data(
&shuffle_disks,
RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET,
&commit_upload_id_path, &commit_upload_id_path,
parts_metadatas, &parts_metadatas,
&commit_bucket, &commit_bucket,
&commit_object, &commit_object,
write_quorum, write_quorum,
) )
.await; .await?;
if quota_mutation_fence {
let _ = SetDisks::release_quota_mutation_fences(
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
)
.await;
}
if rename_result.is_ok() {
quota_reservation.commit().await;
}
let rename_commit = match rename_result {
Ok(result) => result,
Err(err) => return Err(err.into()),
};
let online_disks = rename_commit.online_disks;
let convergence = rename_commit.convergence;
let op_old_dir = rename_commit.data_dir;
let cleanup_disks = rename_commit.cleanup_disks;
let committed_file_info = rename_commit.committed_file_info;
// Detach admission before any post-commit await: client cancellation // Detach admission before any post-commit await: client cancellation
// must not couple durable convergence repair to cleanup work. // must not couple durable convergence repair to cleanup work.
@@ -2612,7 +2420,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
return Err(StorageError::Unexpected); return Err(StorageError::Unexpected);
} }
fi = committed_file_info; if let Some(committed_slot) = online_disks.iter().position(Option::is_some) {
fi = parts_metadatas[committed_slot].clone();
}
let committed_dir = fi.data_dir.unwrap_or_default().to_string(); let committed_dir = fi.data_dir.unwrap_or_default().to_string();
commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks); commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks);
@@ -3117,34 +2927,12 @@ mod tests {
content: &[u8], content: &[u8],
actual_size: i64, actual_size: i64,
) -> CompletePart { ) -> CompletePart {
put_test_part_with_opts(
set_disks,
bucket,
object,
upload_id,
part_number,
(content, actual_size),
&ObjectOptions::default(),
)
.await
}
async fn put_test_part_with_opts(
set_disks: &Arc<SetDisks>,
bucket: &str,
object: &str,
upload_id: &str,
part_number: usize,
part: (&[u8], i64),
opts: &ObjectOptions,
) -> CompletePart {
let (content, actual_size) = part;
let mut reader = PutObjReader::new( let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false) HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false)
.expect("hash reader should be constructed"), .expect("hash reader should be constructed"),
); );
let part = set_disks let part = set_disks
.put_object_part(bucket, object, upload_id, part_number, &mut reader, opts) .put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default())
.await .await
.expect("uploading the part should succeed"); .expect("uploading the part should succeed");
CompletePart { CompletePart {
@@ -3374,7 +3162,7 @@ mod tests {
let (upload_id, parts) = let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await; stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
let mut denied_opts = ObjectOptions::default(); let mut denied_opts = ObjectOptions::default();
assert!(denied_opts.set_quota_admission(100, 4180)); assert!(denied_opts.set_quota_admission(100, 4195));
let err = set_disks let err = set_disks
.clone() .clone()
@@ -3385,7 +3173,7 @@ mod tests {
err, err,
StorageError::QuotaExceeded { StorageError::QuotaExceeded {
current: 100, current: 100,
limit: 4180 limit: 4195
} }
)); ));
@@ -3403,7 +3191,7 @@ mod tests {
); );
let mut allowed_opts = ObjectOptions::default(); let mut allowed_opts = ObjectOptions::default();
assert!(allowed_opts.set_quota_admission(100, 4181)); assert!(allowed_opts.set_quota_admission(100, 4196));
let completed = set_disks let completed = set_disks
.clone() .clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts) .complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
@@ -3444,120 +3232,6 @@ mod tests {
); );
} }
#[tokio::test]
async fn replication_quota_uses_server_observed_part_size_as_lower_bound() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut create_opts = ObjectOptions::default();
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("replication multipart upload should be created");
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await;
let mut complete_opts = ObjectOptions {
replication_request: true,
..Default::default()
};
assert!(complete_opts.set_quota_admission(0, 4095));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts)
.await
.expect_err("a forged tiny replication logical size must not reduce quota admission");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"quota rejection must leave replicated multipart parts retryable"
);
assert!(complete_opts.set_quota_admission(0, 4096));
let completed = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect("the physical safety boundary should admit the transformed replica");
assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1);
}
#[tokio::test]
async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-direct-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await;
let mut complete_opts = ObjectOptions::default();
assert!(complete_opts.set_quota_admission(0, 4095));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("a forged tiny direct logical size must not reduce quota admission");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"quota rejection must leave direct multipart parts retryable"
);
}
#[tokio::test]
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-compressed-ciphertext-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut create_opts = ObjectOptions::default();
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
insert_str(
&mut create_opts.user_defined,
rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT,
"true".to_string(),
);
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("ciphertext multipart upload should be created");
let payload = vec![0x74; 4096];
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await;
let mut complete_opts = ObjectOptions {
replication_request: true,
..Default::default()
};
assert!(complete_opts.set_quota_admission(0, u64::MAX));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("ciphertext replication has no server-observed logical quota size");
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"rejection must leave ciphertext multipart parts retryable"
);
}
#[tokio::test] #[tokio::test]
async fn complete_multipart_quota_rejects_invalid_logical_sizes() { async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -3880,6 +3554,7 @@ mod tests {
async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() { async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "data-movement-unknown-actual-size-bucket"; let bucket = "data-movement-unknown-actual-size-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await; make_bucket_on_all(&disk_stores, bucket).await;
let mut metadata = HashMap::new(); let mut metadata = HashMap::new();
@@ -3893,63 +3568,6 @@ mod tests {
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"source-generation".to_string(), "source-generation".to_string(),
); );
for (object, quota_limit) in [("without-quota", None), ("with-quota", Some(u64::MAX))] {
let create_opts = ObjectOptions {
data_movement: true,
user_defined: metadata.clone(),
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let mut completed_parts = Vec::new();
for (number, actual_size) in [(1, -1), (2, 1)] {
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false)
.expect("part reader should be constructed"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts)
.await
.expect("data movement part should be written");
completed_parts.push(CompletePart {
part_num: number,
etag: part.etag,
..Default::default()
});
}
let missing_size_err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts.clone(), &create_opts)
.await
.expect_err("data movement completion must require an authoritative total size");
assert!(matches!(missing_size_err, StorageError::PartMissingOrCorrupt));
let mut complete_opts = create_opts.clone();
rustfs_utils::http::insert_str(
&mut complete_opts.user_defined,
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
"2".to_string(),
);
if let Some(quota_limit) = quota_limit {
assert!(complete_opts.set_quota_admission(0, quota_limit));
}
let completion = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &complete_opts)
.await;
let completed = completion.expect("data movement completion should accept the persisted unknown-size sentinel");
assert_eq!(completed.parts[0].actual_size, -1);
assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2);
}
let object = "all-unknown-with-quota";
let create_opts = ObjectOptions { let create_opts = ObjectOptions {
data_movement: true, data_movement: true,
user_defined: metadata.clone(), user_defined: metadata.clone(),
@@ -3959,93 +3577,43 @@ mod tests {
.new_multipart_upload(bucket, object, &create_opts) .new_multipart_upload(bucket, object, &create_opts)
.await .await
.expect("data movement upload should be created"); .expect("data movement upload should be created");
let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x40], -1), &create_opts).await;
let mut complete_opts = create_opts; let mut completed_parts = Vec::new();
rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); for (number, actual_size) in [(1, -1), (2, 1)] {
assert!(complete_opts.set_quota_admission(0, u64::MAX)); let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false)
.expect("part reader should be constructed"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts)
.await
.expect("data movement part should be written");
completed_parts.push(CompletePart {
part_num: number,
etag: part.etag,
..Default::default()
});
}
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string());
let completed = set_disks let completed = set_disks
.clone() .clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) .complete_multipart_upload(
bucket,
object,
&upload.upload_id,
completed_parts,
&ObjectOptions {
data_movement: true,
user_defined: metadata,
..Default::default()
},
)
.await .await
.expect("quota must accept an all-unknown data movement upload with an authoritative total"); .expect("data movement completion should accept the persisted unknown-size sentinel");
assert_eq!(completed.parts[0].actual_size, -1); assert_eq!(completed.parts[0].actual_size, -1);
assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2);
let object = "legacy-zero-fallback";
let create_opts = ObjectOptions {
data_movement: true,
user_defined: metadata.clone(),
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let part =
put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41; 128], 128), &create_opts).await;
let mut complete_opts = create_opts;
rustfs_utils::http::insert_str(
&mut complete_opts.user_defined,
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
"100".to_string(),
);
assert!(complete_opts.set_quota_admission(0, u64::MAX));
let completed = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect("legacy zero-to-physical part fallback must remain migratable");
assert_eq!(completed.parts[0].actual_size, 128);
assert_eq!(completed.get_actual_size().expect("completed object actual size"), 100);
let object = "inconsistent-untransformed-total";
let mut untransformed_metadata = metadata.clone();
rustfs_utils::http::remove_str(&mut untransformed_metadata, rustfs_utils::http::SUFFIX_COMPRESSION);
let create_opts = ObjectOptions {
data_movement: true,
user_defined: untransformed_metadata,
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x42; 2], 2), &create_opts).await;
let mut complete_opts = create_opts;
rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("authoritative total below known logical part sizes must fail closed");
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
for (object, declared_size) in [("invalid-total", "invalid"), ("negative-total", "-1")] {
let create_opts = ObjectOptions {
data_movement: true,
user_defined: metadata.clone(),
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let part =
put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41], 1), &create_opts).await;
let mut complete_opts = create_opts.clone();
rustfs_utils::http::insert_str(
&mut complete_opts.user_defined,
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
declared_size.to_string(),
);
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("invalid authoritative total size must fail closed");
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
}
} }
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) { async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
File diff suppressed because it is too large Load Diff
-26
View File
@@ -1174,32 +1174,6 @@ mod tests {
} }
} }
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
let (_ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
let object = "object.bin";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create quota fence bucket");
store.pools[1].disk_set[0].disks.write().await.fill(None);
crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
.await
.expect("the selected pool fence should ignore an unrelated offline pool");
let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
.await
.expect_err("legacy reservations must conservatively fence every pool");
assert!(matches!(err, StorageError::ErasureWriteQuorum));
shutdown.cancel();
}
#[tokio::test] #[tokio::test]
#[serial_test::serial(storage_class_env)] #[serial_test::serial(storage_class_env)]
async fn tag_updates_skip_active_rebalance_source_pool() { async fn tag_updates_skip_active_rebalance_source_pool() {
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
impl ECStore {
#[instrument(level = "trace", skip(self))]
#[allow(clippy::too_many_arguments)]
pub(super) async fn handle_list_objects_v2(
self: Arc<Self>,
bucket: &str,
prefix: &str,
continuation_token: Option<String>,
delimiter: Option<String>,
max_keys: i32,
fetch_owner: bool,
start_after: Option<String>,
incl_deleted: bool,
) -> Result<ListObjectsV2Info> {
self.inner_list_objects_v2(
bucket,
prefix,
continuation_token,
delimiter,
max_keys,
fetch_owner,
start_after,
incl_deleted,
)
.await
}
#[instrument(skip(self))]
pub(super) async fn handle_list_object_versions(
self: Arc<Self>,
bucket: &str,
prefix: &str,
marker: Option<String>,
version_marker: Option<String>,
delimiter: Option<String>,
max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
.await
}
pub(crate) async fn list_object_versions_for_lifecycle(
self: Arc<Self>,
bucket: &str,
prefix: &str,
marker: Option<String>,
version_marker: Option<String>,
delimiter: Option<String>,
max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
self.inner_list_object_versions_for_lifecycle(bucket, prefix, marker, version_marker, delimiter, max_keys)
.await
}
pub(super) async fn handle_walk(
self: Arc<Self>,
rx: CancellationToken,
bucket: &str,
prefix: &str,
result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
opts: WalkOptions,
) -> Result<()> {
self.walk_internal(rx, bucket, prefix, result, opts).await
}
}
+2 -2
View File
@@ -641,7 +641,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta))); .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
} }
pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
} }
@@ -3845,7 +3845,7 @@ impl ECStore {
.await .await
} }
pub(crate) async fn list_object_versions_for_lifecycle( pub(crate) async fn inner_list_object_versions_for_lifecycle(
self: Arc<Self>, self: Arc<Self>,
bucket: &str, bucket: &str,
prefix: &str, prefix: &str,
+4 -3
View File
@@ -148,6 +148,7 @@ mod heal_walk;
pub use heal_walk::HealWalkVersion; pub use heal_walk::HealWalkVersion;
mod init; mod init;
pub(crate) mod init_format; pub(crate) mod init_format;
mod list;
pub(crate) mod list_objects; pub(crate) mod list_objects;
mod multipart; mod multipart;
mod object; mod object;
@@ -600,7 +601,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore {
start_after: Option<String>, start_after: Option<String>,
incl_deleted: bool, incl_deleted: bool,
) -> Result<ListObjectsV2Info> { ) -> Result<ListObjectsV2Info> {
self.inner_list_objects_v2( self.handle_list_objects_v2(
bucket, bucket,
prefix, prefix,
continuation_token, continuation_token,
@@ -623,7 +624,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore {
delimiter: Option<String>, delimiter: Option<String>,
max_keys: i32, max_keys: i32,
) -> Result<ListObjectVersionsInfo> { ) -> Result<ListObjectVersionsInfo> {
self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys) self.handle_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
.await .await
} }
@@ -635,7 +636,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore {
result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>, result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
opts: WalkOptions, opts: WalkOptions,
) -> Result<()> { ) -> Result<()> {
self.walk_internal(rx, bucket, prefix, result, opts).await self.handle_walk(rx, bucket, prefix, result, opts).await
} }
} }
+3 -4
View File
@@ -4741,10 +4741,9 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial_test::serial(body_cache_hook)] #[serial_test::serial(body_cache_hook)]
async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() { async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await;
let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
let bucket = "select-snapshot-latest-delete-marker"; let bucket = "select-snapshot-latest-delete-marker";
let object = "versioned-object.bin"; let object = "versioned-object.bin";
let versioned_opts = ObjectOptions { let versioned_opts = ObjectOptions {
@@ -157,18 +157,6 @@ fn ecstore_implements_storage_list_operations_contract() {
assert!(storage_list_operations_type_name::<ECStore>().ends_with("::ECStore")); assert!(storage_list_operations_type_name::<ECStore>().ends_with("::ECStore"));
} }
#[test]
fn ecstore_pools_expose_storage_list_operations_contract() {
fn assert_contract(store: &ECStore) {
let future = store.pools[0]
.clone()
.list_objects_v2("bucket", "", None, None, 1, false, None, false);
drop(future);
}
let _ = assert_contract;
}
#[test] #[test]
fn ecstore_implements_storage_multipart_operations_contract() { fn ecstore_implements_storage_multipart_operations_contract() {
assert!(storage_multipart_operations_type_name::<ECStore>().ends_with("::ECStore")); assert!(storage_multipart_operations_type_name::<ECStore>().ends_with("::ECStore"));
+4 -1
View File
@@ -135,7 +135,10 @@ impl FileMeta {
let i = buf.len() as u64; let i = buf.len() as u64;
// check version, buf = buf[8..] // check version, buf = buf[8..]
let (buf, _, _) = Self::check_xl2_v1(buf)?; let (buf, _, _) = Self::check_xl2_v1(buf).map_err(|e| {
error!("failed to check XL2 v1 format: {}", e);
e
})?;
if buf.len() < 5 { if buf.len() < 5 {
error!( error!(
+2
View File
@@ -225,6 +225,8 @@ async fn nothing_readable_leaves_the_bundle_unwrapped() {
"artifact {} carries the raw on-disk record", "artifact {} carries the raw on-disk record",
artifact.path artifact.path
); );
// A cheap structural check too: an encrypted payload is not JSON.
assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path);
} }
// The manifest itself is not encrypted, so assert directly that it carries // The manifest itself is not encrypted, so assert directly that it carries
+5 -1
View File
@@ -103,7 +103,11 @@ pub(super) fn rules() -> Vec<Rule> {
P2Degraded, P2Degraded,
"heal", "heal",
"heal 任务调度/执行失败", "heal 任务调度/执行失败",
any([prefix("Heal task timeout"), prefix("Heal task execution failed")]), any([
prefix("Heal task timeout"),
prefix("Heal task execution failed"),
contains("Heal manager is not running"),
]),
"heal 任务调度/执行层故障。", "heal 任务调度/执行层故障。",
"检查 heal 后台服务状态与资源压力。", "检查 heal 后台服务状态与资源压力。",
) )
+18 -24
View File
@@ -748,10 +748,9 @@ async fn handle_authenticated_request(
} }
for (key, value) in info.user_defined.iter() { for (key, value) in info.user_defined.iter() {
if key != "content-type" if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
@@ -818,10 +817,9 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*) // Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() { for (key, value) in info.user_defined.iter() {
if key != "content-type" if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
@@ -858,10 +856,9 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*) // Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() { for (key, value) in info.user_defined.iter() {
if key != "content-type" if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
@@ -1171,10 +1168,9 @@ async fn handle_object_get(
} }
for (key, value) in info.user_defined.iter() { for (key, value) in info.user_defined.iter() {
if key != "content-type" if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
@@ -1241,10 +1237,9 @@ async fn handle_object_get(
if key == "x-delete-at" { if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*) // Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str()); response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" } else if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
@@ -1298,10 +1293,9 @@ async fn handle_object_head(
if key == "x-delete-at" { if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*) // Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str()); response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" } else if key != "content-type" {
&& let Some(key) = object::swift_response_user_metadata_key(key) let header_name = format!("x-object-meta-{}", key);
{ response = response.header(header_name, value.as_str());
response = response.header(format!("x-object-meta-{key}"), value.as_str());
} }
} }
+31 -81
View File
@@ -67,57 +67,10 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object"; const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state"; const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at"; const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
const USER_METADATA_PREFIX: &str = "x-amz-meta-";
/// Maximum object size in bytes (5GB - Swift default) /// Maximum object size in bytes (5GB - Swift default)
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024; const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
fn stored_swift_user_metadata_key(key: &str) -> String {
if rustfs_utils::http::is_internal_key(key)
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
{
format!("{USER_METADATA_PREFIX}{key}")
} else {
key.to_string()
}
}
pub(super) fn swift_response_user_metadata_key(key: &str) -> Option<&str> {
if rustfs_utils::http::is_internal_key(key)
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
{
return None;
}
if let Some(unescaped) = key.strip_prefix(USER_METADATA_PREFIX)
&& (rustfs_utils::http::is_internal_key(unescaped)
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-amz-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-minio-encryption-"))
{
return Some(unescaped);
}
Some(key)
}
fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
let mut metadata = HashMap::new();
let mut present = false;
for (header_name, header_value) in headers.iter() {
let header_name = header_name.as_str().to_lowercase();
let Some(key) = header_name.strip_prefix("x-object-meta-") else {
continue;
};
present = true;
if let Ok(value) = header_value.to_str() {
metadata.insert(stored_swift_user_metadata_key(key), value.to_string());
}
}
present.then_some(metadata)
}
/// Object key translator for Swift object names /// Object key translator for Swift object names
/// ///
/// Handles URL encoding/decoding and path normalization for Swift object keys. /// Handles URL encoding/decoding and path normalization for Swift object keys.
@@ -350,7 +303,15 @@ where
let bucket = mapper.swift_to_s3_bucket(container, &project_id); let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Extract Swift metadata from X-Object-Meta-* headers // 5. Extract Swift metadata from X-Object-Meta-* headers
let mut user_metadata = swift_user_metadata(headers).unwrap_or_default(); let mut user_metadata = HashMap::new();
for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
&& let Ok(value_str) = header_value.to_str()
{
user_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
// 6. Extract Content-Type if provided // 6. Extract Content-Type if provided
if let Some(content_type) = headers.get("content-type") if let Some(content_type) = headers.get("content-type")
@@ -778,7 +739,15 @@ pub async fn update_object_metadata(
} }
// 8. Extract new metadata from X-Object-Meta-* headers // 8. Extract new metadata from X-Object-Meta-* headers
let mut new_metadata = swift_user_metadata(headers).unwrap_or_default(); let mut new_metadata = HashMap::new();
for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
&& let Ok(value_str) = header_value.to_str()
{
new_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
// 9. Also update Content-Type if provided // 9. Also update Content-Type if provided
if let Some(content_type) = headers.get("content-type") if let Some(content_type) = headers.get("content-type")
@@ -920,8 +889,19 @@ pub async fn copy_object(
let mut new_metadata = (*src_info.user_defined).clone(); let mut new_metadata = (*src_info.user_defined).clone();
// 11. If custom metadata headers provided, use those instead (Swift behavior) // 11. If custom metadata headers provided, use those instead (Swift behavior)
if let Some(custom_metadata) = swift_user_metadata(headers) { let mut has_custom_meta = false;
new_metadata = custom_metadata; for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") {
if !has_custom_meta {
// First custom meta header - clear source metadata
new_metadata.clear();
has_custom_meta = true;
}
if let Ok(value_str) = header_value.to_str() {
new_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
} }
// 12. Also check for Content-Type override // 12. Also check for Content-Type override
@@ -1143,36 +1123,6 @@ mod tests {
assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok()); assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok());
} }
#[test]
fn swift_user_metadata_cannot_materialize_internal_storage_keys() {
let mut headers = HeaderMap::new();
headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value"));
headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value"));
let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected");
assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1"));
assert_eq!(metadata.get("description").map(String::as_str), Some("safe"));
assert!(!metadata.contains_key("x-rustfs-internal-actual-size"));
assert_eq!(
stored_swift_user_metadata_key("x-minio-encryption-original-size"),
"x-amz-meta-x-minio-encryption-original-size"
);
assert_eq!(stored_swift_user_metadata_key("description"), "description");
}
#[test]
fn swift_user_metadata_response_mapping_is_reversible_and_filters_internal_keys() {
assert_eq!(
swift_response_user_metadata_key("x-amz-meta-x-rustfs-internal-actual-size"),
Some("x-rustfs-internal-actual-size")
);
assert_eq!(swift_response_user_metadata_key("x-amz-meta-x-amz-checksum"), Some("x-amz-checksum"));
assert_eq!(swift_response_user_metadata_key("x-amz-meta-description"), Some("x-amz-meta-description"));
assert_eq!(swift_response_user_metadata_key("description"), Some("description"));
assert_eq!(swift_response_user_metadata_key("x-rustfs-internal-actual-size"), None);
assert_eq!(swift_response_user_metadata_key("x-minio-internal-actual-size"), None);
}
#[test] #[test]
fn test_validate_object_name_empty() { fn test_validate_object_name_empty() {
let result = ObjectKeyMapper::validate_object_name(""); let result = ObjectKeyMapper::validate_object_name("");
+51 -361
View File
@@ -71,7 +71,6 @@ where
/// Optional: allow users to customize block_size /// Optional: allow users to customize block_size
pub fn with_block_size(inner: R, block_size: usize, compression_algorithm: CompressionAlgorithm) -> Self { pub fn with_block_size(inner: R, block_size: usize, compression_algorithm: CompressionAlgorithm) -> Self {
debug_assert!(block_size > 0, "CompressReader block_size must be non-zero");
Self { Self {
inner, inner,
buffer: Vec::new(), buffer: Vec::new(),
@@ -184,21 +183,11 @@ pin_project! {
buffer: Vec<u8>, buffer: Vec<u8>,
buffer_pos: usize, buffer_pos: usize,
finished: bool, finished: bool,
// A previously surfaced stream error is sticky: without this, a caller
// that polls again after an error would restart at the header phase and
// read a truncated tail as a clean EOF, converting the error into a
// silently short body.
poisoned: bool,
// Fields for saving header read progress across polls // Fields for saving header read progress across polls
header_buf: [u8; 8], header_buf: [u8; 8],
header_read: usize, header_read: usize,
// Fields for saving compressed block read progress across polls. header_done: bool,
// `compressed_len > 0` means a block payload is in flight: the header has // Fields for saving compressed block read progress across polls
// been fully parsed and `compressed_read` bytes of the payload are already
// consumed from the inner stream. The header phase must not run again (and
// must not reset `compressed_read`) until this block completes, or a
// `Poll::Pending` in the middle of a payload would silently drop the bytes
// read so far and desynchronize the block framing.
compressed_buf: Vec<u8>, compressed_buf: Vec<u8>,
compressed_read: usize, compressed_read: usize,
compressed_len: usize, compressed_len: usize,
@@ -216,9 +205,9 @@ where
buffer: Vec::new(), buffer: Vec::new(),
buffer_pos: 0, buffer_pos: 0,
finished: false, finished: false,
poisoned: false,
header_buf: [0u8; 8], header_buf: [0u8; 8],
header_read: 0, header_read: 0,
header_done: false,
compressed_buf: Vec::new(), compressed_buf: Vec::new(),
compressed_read: 0, compressed_read: 0,
compressed_len: 0, compressed_len: 0,
@@ -247,74 +236,54 @@ where
if *this.finished { if *this.finished {
return Poll::Ready(Ok(())); return Poll::Ready(Ok(()));
} }
if *this.poisoned { // Read header
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "decompress reader previously failed"))); while !*this.header_done && *this.header_read < HEADER_LEN {
} let mut temp = [0u8; HEADER_LEN];
let mut temp_buf = ReadBuf::new(&mut temp[0..HEADER_LEN - *this.header_read]);
if *this.compressed_len == 0 { match this.inner.as_mut().poll_read(cx, &mut temp_buf) {
// Read the 8-byte block header, resuming across polls via `header_read`. Poll::Pending => return Poll::Pending,
while *this.header_read < HEADER_LEN { Poll::Ready(Ok(())) => {
let mut temp = [0u8; HEADER_LEN]; let n = temp_buf.filled().len();
let mut temp_buf = ReadBuf::new(&mut temp[0..HEADER_LEN - *this.header_read]); if n == 0 {
match this.inner.as_mut().poll_read(cx, &mut temp_buf) { break;
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = temp_buf.filled().len();
if n == 0 {
if *this.header_read == 0 {
// Clean EOF on a block boundary.
*this.finished = true;
return Poll::Ready(Ok(()));
}
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading compressed block header",
)));
}
this.header_buf[*this.header_read..*this.header_read + n].copy_from_slice(&temp_buf.filled()[..n]);
*this.header_read += n;
}
Poll::Ready(Err(e)) => {
// error!("DecompressReader poll_read: read header error: {e}");
*this.poisoned = true;
return Poll::Ready(Err(e));
} }
this.header_buf[*this.header_read..*this.header_read + n].copy_from_slice(&temp_buf.filled()[..n]);
*this.header_read += n;
}
Poll::Ready(Err(e)) => {
// error!("DecompressReader poll_read: read header error: {e}");
return Poll::Ready(Err(e));
} }
} }
if *this.header_read < HEADER_LEN {
return Poll::Pending;
}
}
if !*this.header_done && *this.header_read == 0 {
return Poll::Ready(Ok(()));
}
let typ = this.header_buf[0];
let len = (this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16);
let crc = (this.header_buf[4] as u32)
| ((this.header_buf[5] as u32) << 8)
| ((this.header_buf[6] as u32) << 16)
| ((this.header_buf[7] as u32) << 24);
*this.header_read = 0;
*this.header_done = true;
let typ = this.header_buf[0]; if typ == COMPRESS_TYPE_END {
let len =
(this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16);
*this.header_read = 0;
// `CompressReader` never emits an end block — a stream terminates on
// inner EOF, which is what lets concatenated per-part streams decode as
// one. This branch is kept for streams that do carry the marker.
if typ == COMPRESS_TYPE_END {
*this.compressed_read = 0;
*this.compressed_len = 0;
*this.finished = true;
return Poll::Ready(Ok(()));
}
if typ != COMPRESS_TYPE_COMPRESSED && typ != COMPRESS_TYPE_UNCOMPRESSED {
// error!("DecompressReader unknown compression type: {typ}");
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown compression type")));
}
if len == 0 {
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid compressed block length")));
}
if this.compressed_buf.len() < len {
this.compressed_buf.resize(len, 0);
}
*this.compressed_len = len;
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0;
*this.finished = true;
return Poll::Ready(Ok(()));
} }
// Fill the in-flight block payload, resuming across polls via `compressed_read`. if this.compressed_buf.len() < len {
this.compressed_buf.resize(len, 0);
}
*this.compressed_len = len;
*this.compressed_read = 0;
while *this.compressed_read < *this.compressed_len { while *this.compressed_read < *this.compressed_len {
let mut temp_buf = ReadBuf::new(&mut this.compressed_buf[*this.compressed_read..*this.compressed_len]); let mut temp_buf = ReadBuf::new(&mut this.compressed_buf[*this.compressed_read..*this.compressed_len]);
match this.inner.as_mut().poll_read(cx, &mut temp_buf) { match this.inner.as_mut().poll_read(cx, &mut temp_buf) {
@@ -322,13 +291,7 @@ where
Poll::Ready(Ok(())) => { Poll::Ready(Ok(())) => {
let n = temp_buf.filled().len(); let n = temp_buf.filled().len();
if n == 0 { if n == 0 {
*this.compressed_read = 0; break;
*this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading compressed block payload",
)));
} }
*this.compressed_read += n; *this.compressed_read += n;
} }
@@ -336,17 +299,10 @@ where
// error!("DecompressReader poll_read: read compressed block error: {e}"); // error!("DecompressReader poll_read: read compressed block error: {e}");
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(e)); return Poll::Ready(Err(e));
} }
} }
} }
let typ = this.header_buf[0];
let crc = (this.header_buf[4] as u32)
| ((this.header_buf[5] as u32) << 8)
| ((this.header_buf[6] as u32) << 16)
| ((this.header_buf[7] as u32) << 24);
let compressed_buf = &this.compressed_buf[..*this.compressed_len]; let compressed_buf = &this.compressed_buf[..*this.compressed_len];
// `compressed_buf`'s length comes from the untrusted 24-bit header length field, so it // `compressed_buf`'s length comes from the untrusted 24-bit header length field, so it
// can be shorter than 16 bytes. `uvarint` is safe on any slice length (reads at most 10 // can be shorter than 16 bytes. `uvarint` is safe on any slice length (reads at most 10
@@ -360,7 +316,6 @@ where
if uvarint <= 0 || uvarint as usize > compressed_buf.len() { if uvarint <= 0 || uvarint as usize > compressed_buf.len() {
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid compressed block length prefix"))); return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid compressed block length prefix")));
} }
let compressed_data = &compressed_buf[uvarint as usize..]; let compressed_data = &compressed_buf[uvarint as usize..];
@@ -371,29 +326,21 @@ where
// error!("DecompressReader decompress_block error: {e}"); // error!("DecompressReader decompress_block error: {e}");
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(e)); return Poll::Ready(Err(e));
} }
} }
} else { } else if typ == COMPRESS_TYPE_UNCOMPRESSED {
// The header phase already rejected every type other than
// COMPRESS_TYPE_COMPRESSED / COMPRESS_TYPE_UNCOMPRESSED.
compressed_data.to_vec() compressed_data.to_vec()
}; } else {
if decompressed.is_empty() { // error!("DecompressReader unknown compression type: {typ}");
// The writer never emits zero-length plaintext blocks; an empty
// decode surfacing as Ready(Ok) with no bytes would read as EOF and
// silently truncate the stream.
*this.poisoned = true;
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Empty compressed block"))); return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown compression type")));
} };
if decompressed.len() != uncompress_len as usize { if decompressed.len() != uncompress_len as usize {
// error!("DecompressReader decompressed length mismatch: {} != {}", decompressed.len(), uncompress_len); // error!("DecompressReader decompressed length mismatch: {} != {}", decompressed.len(), uncompress_len);
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Decompressed length mismatch"))); return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Decompressed length mismatch")));
} }
let actual_crc = { let actual_crc = {
@@ -405,13 +352,13 @@ where
// error!("DecompressReader CRC32 mismatch: actual {actual_crc} != expected {crc}"); // error!("DecompressReader CRC32 mismatch: actual {actual_crc} != expected {crc}");
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.poisoned = true;
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "CRC32 mismatch"))); return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "CRC32 mismatch")));
} }
*this.buffer = decompressed; *this.buffer = decompressed;
*this.buffer_pos = 0; *this.buffer_pos = 0;
*this.compressed_read = 0; *this.compressed_read = 0;
*this.compressed_len = 0; *this.compressed_len = 0;
*this.header_done = false;
let to_copy = min(buf.remaining(), this.buffer.len()); let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]); buf.put_slice(&this.buffer[..to_copy]);
*this.buffer_pos += to_copy; *this.buffer_pos += to_copy;
@@ -546,184 +493,6 @@ mod tests {
assert_eq!(&decompressed, &data); assert_eq!(&decompressed, &data);
} }
/// Wraps a reader so every other poll returns `Poll::Pending` and every
/// `Ready` poll serves at most `chunk` bytes. This is the shape a duplex
/// pipe produces when the erasure writer is slower than the decoder, which
/// is exactly what desynchronized the block framing before the resumable
/// payload state was added (rustfs/rustfs#5957 multipart GET truncation).
struct PendingChunkReader<R> {
inner: R,
chunk: usize,
pending_next: bool,
}
impl<R> PendingChunkReader<R> {
fn new(inner: R, chunk: usize) -> Self {
Self {
inner,
chunk,
pending_next: true,
}
}
}
impl<R: AsyncRead + Unpin> AsyncRead for PendingChunkReader<R> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
if self.pending_next {
self.pending_next = false;
cx.waker().wake_by_ref();
return std::task::Poll::Pending;
}
self.pending_next = true;
let cap = self.chunk.min(buf.remaining());
let mut scratch = vec![0u8; cap];
let mut inner_buf = tokio::io::ReadBuf::new(&mut scratch);
match std::pin::Pin::new(&mut self.inner).poll_read(cx, &mut inner_buf) {
std::task::Poll::Ready(Ok(())) => {
buf.put_slice(inner_buf.filled());
std::task::Poll::Ready(Ok(()))
}
other => other,
}
}
}
fn patterned_payload(size: usize, seed: u8) -> Vec<u8> {
(0..size)
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
.collect()
}
/// Root-cause regression for the multipart compressed GET truncation: a
/// `Poll::Pending` in the middle of a block payload must not drop the bytes
/// already consumed. Before the resumable payload state, the decoder reset
/// `compressed_read` on every re-poll and surfaced
/// `LZ4 error: ERROR_frameType_unknown` mid-stream.
#[tokio::test]
async fn test_decompress_reader_survives_pending_mid_payload() {
let data = patterned_payload(100 * 1024, 7);
let mut compress_reader =
CompressReader::with_block_size(Cursor::new(data.clone()), 8192, CompressionAlgorithm::default());
let mut compressed = Vec::new();
compress_reader.read_to_end(&mut compressed).await.unwrap();
for chunk in [1usize, 3, 7, 8, 17, 1000, 8192] {
let inner = PendingChunkReader::new(Cursor::new(compressed.clone()), chunk);
let mut decompress_reader = DecompressReader::new(inner, CompressionAlgorithm::default());
let mut decompressed = Vec::new();
decompress_reader.read_to_end(&mut decompressed).await.unwrap();
assert_eq!(decompressed, data, "pending-chunked decode must be byte-exact for chunk={chunk}");
}
}
/// Two independently compressed streams concatenated back to back — the
/// on-disk shape of a compressed multipart object — must decode across the
/// stream boundary even when every poll can suspend mid-block.
#[tokio::test]
async fn test_decompress_reader_survives_pending_across_concatenated_streams() {
let part1 = patterned_payload(64 * 1024, 7);
let part2 = patterned_payload(24 * 1024, 61);
let mut stored = Vec::new();
for part in [&part1, &part2] {
let mut compress_reader =
CompressReader::with_block_size(Cursor::new(part.clone()), 8192, CompressionAlgorithm::default());
let mut compressed = Vec::new();
compress_reader.read_to_end(&mut compressed).await.unwrap();
stored.extend_from_slice(&compressed);
}
let mut expected = part1;
expected.extend_from_slice(&part2);
for chunk in [1usize, 5, 8, 13, 4096] {
let inner = PendingChunkReader::new(Cursor::new(stored.clone()), chunk);
let mut decompress_reader = DecompressReader::new(inner, CompressionAlgorithm::default());
let mut decompressed = Vec::new();
decompress_reader.read_to_end(&mut decompressed).await.unwrap();
assert_eq!(
decompressed, expected,
"concatenated part streams must decode byte-exact for chunk={chunk}"
);
}
}
/// After the first stream error, every further poll must keep failing.
/// Without the sticky poison a retrying caller would restart at the header
/// phase and read the truncated tail as a clean EOF — converting a hard
/// error into a silently short body.
#[tokio::test]
async fn test_decompress_reader_error_is_sticky() {
let data = patterned_payload(32 * 1024, 7);
let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default());
let mut compressed = Vec::new();
compress_reader.read_to_end(&mut compressed).await.unwrap();
compressed.truncate(compressed.len() - 3);
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut out = Vec::new();
let first = decompress_reader
.read_to_end(&mut out)
.await
.expect_err("truncated payload must error");
assert_eq!(first.kind(), std::io::ErrorKind::UnexpectedEof);
let mut retry = Vec::new();
let second = decompress_reader
.read_to_end(&mut retry)
.await
.expect_err("a poll after the first error must not turn into a clean EOF");
assert_eq!(second.kind(), std::io::ErrorKind::InvalidData);
assert!(retry.is_empty(), "no bytes may be produced after the stream failed");
}
/// A stream cut off in the middle of a block payload must fail with a clean
/// UnexpectedEof instead of decoding a short buffer.
#[tokio::test]
async fn test_decompress_reader_truncated_payload_is_unexpected_eof() {
let data = patterned_payload(32 * 1024, 7);
let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default());
let mut compressed = Vec::new();
compress_reader.read_to_end(&mut compressed).await.unwrap();
compressed.truncate(compressed.len() - 3);
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut out = Vec::new();
let err = decompress_reader
.read_to_end(&mut out)
.await
.expect_err("truncated payload must error");
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
}
/// A stream cut off in the middle of a block header must fail with a clean
/// UnexpectedEof instead of parsing a garbage header.
#[tokio::test]
async fn test_decompress_reader_truncated_header_is_unexpected_eof() {
let data = patterned_payload(12 * 1024, 7);
let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default());
let mut compressed = Vec::new();
compress_reader.read_to_end(&mut compressed).await.unwrap();
// Keep the first full block plus 3 bytes of the next header.
let ln = (compressed[1] as usize) | ((compressed[2] as usize) << 8) | ((compressed[3] as usize) << 16);
let first_block_end = 8 + ln;
assert!(compressed.len() > first_block_end, "fixture must contain more than one block");
compressed.truncate(first_block_end + 3);
let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut out = Vec::new();
let err = decompress_reader
.read_to_end(&mut out)
.await
.expect_err("truncated header must error");
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
}
// Regression: a corrupted block whose 24-bit length field is < 16 must not panic. // Regression: a corrupted block whose 24-bit length field is < 16 must not panic.
// Header layout (HEADER_LEN = 8): [type, len_lo, len_mid, len_hi, crc0..crc3], then `len` // Header layout (HEADER_LEN = 8): [type, len_lo, len_mid, len_hi, crc0..crc3], then `len`
// bytes of block body. Pre-fix, poll_read sliced `compressed_buf[0..16]` unconditionally, // bytes of block body. Pre-fix, poll_read sliced `compressed_buf[0..16]` unconditionally,
@@ -749,85 +518,6 @@ mod tests {
assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData); assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
} }
// Header-level fail-closed matrix, built by hand so the decoder is exercised against bytes no
// encoder in this crate can produce. Header layout (HEADER_LEN = 8):
// [type, len_lo, len_mid, len_hi, crc0..crc3], then `len` body bytes = uvarint(plain_len) + data.
#[tokio::test]
async fn test_decompress_reader_header_validation_matrix() {
// Build a block whose body is `uvarint(plain.len()) + plain` (i.e. the
// COMPRESS_TYPE_UNCOMPRESSED shape), with the header CRC taken over the plaintext exactly
// like the production writer does.
fn build_raw_block(typ: u8, plain: &[u8], len_override: Option<usize>) -> Vec<u8> {
let crc = {
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(plain);
hasher.finalize() as u32
};
let mut uvarint_buf = [0u8; 10];
let int_len = put_uvarint(&mut uvarint_buf[..], plain.len() as u64);
let body_len = int_len + plain.len();
let len = len_override.unwrap_or(body_len);
let mut out = Vec::with_capacity(HEADER_LEN + body_len);
out.push(typ);
out.push((len & 0xFF) as u8);
out.push(((len >> 8) & 0xFF) as u8);
out.push(((len >> 16) & 0xFF) as u8);
out.extend_from_slice(&crc.to_le_bytes());
out.extend_from_slice(&uvarint_buf[..int_len]);
out.extend_from_slice(plain);
out
}
let plain = b"uncompressed passthrough payload";
// (a) A well-formed uncompressed block decodes to the plaintext verbatim.
let mut out = Vec::new();
DecompressReader::new(
Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, plain, None)),
CompressionAlgorithm::default(),
)
.read_to_end(&mut out)
.await
.expect("a well-formed uncompressed block must decode");
assert_eq!(out.as_slice(), plain.as_slice());
// (b) An unknown block type must be rejected instead of being treated as passthrough.
let mut out = Vec::new();
let err = DecompressReader::new(Cursor::new(build_raw_block(0x7E, plain, None)), CompressionAlgorithm::default())
.read_to_end(&mut out)
.await
.expect_err("unknown compression type must error");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("Unknown compression type"), "got: {err}");
// (c) A zero-length block would stall the decoder, so it must be rejected up front.
let mut out = Vec::new();
let err = DecompressReader::new(
Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, plain, Some(0))),
CompressionAlgorithm::default(),
)
.read_to_end(&mut out)
.await
.expect_err("zero-length block must error");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("Invalid compressed block length"), "got: {err}");
// (d) A block that decodes to zero plaintext bytes must be rejected: the
// writer never emits empty blocks, and an empty decode surfacing as
// Ready(Ok) with no bytes would read as EOF and silently truncate.
let mut out = Vec::new();
let err = DecompressReader::new(
Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, b"", None)),
CompressionAlgorithm::default(),
)
.read_to_end(&mut out)
.await
.expect_err("empty block must error");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("Empty compressed block"), "got: {err}");
}
// Directly exercises the length-prefix guard: an unterminated varint (all continuation bytes) // Directly exercises the length-prefix guard: an unterminated varint (all continuation bytes)
// makes `uvarint` return 0, which must be rejected as an invalid length prefix. // makes `uvarint` return 0, which must be rejected as an invalid length prefix.
#[tokio::test] #[tokio::test]
+28 -81
View File
@@ -24,17 +24,12 @@ pin_project! {
#[pin] #[pin]
pub inner: R, pub inner: R,
remaining: i64, remaining: i64,
scratch: Vec<u8>,
} }
} }
impl<R> HardLimitReader<R> { impl<R> HardLimitReader<R> {
pub fn new(inner: R, limit: i64) -> Self { pub fn new(inner: R, limit: i64) -> Self {
HardLimitReader { HardLimitReader { inner, remaining: limit }
inner,
remaining: limit,
scratch: Vec::new(),
}
} }
} }
@@ -42,21 +37,19 @@ impl<R> AsyncRead for HardLimitReader<R>
where where
R: AsyncRead, R: AsyncRead,
{ {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<Result<()>> { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<Result<()>> {
let mut this = self.project(); if self.remaining < 0 {
if *this.remaining < 0 {
return Poll::Ready(Err(Error::other("input provided more bytes than specified"))); return Poll::Ready(Err(Error::other("input provided more bytes than specified")));
} }
if buf.remaining() == 0 { let original_filled = buf.filled().len();
return Poll::Ready(Ok(())); if self.remaining == 0 {
}
if *this.remaining == 0 {
let mut discard = [0u8; 8192]; let mut discard = [0u8; 8192];
let mut discard_buf = ReadBuf::new(&mut discard); let mut discard_buf = ReadBuf::new(&mut discard);
return match this.inner.as_mut().poll_read(cx, &mut discard_buf) { return match self.as_mut().project().inner.poll_read(cx, &mut discard_buf) {
Poll::Pending => Poll::Pending, Poll::Pending => Poll::Pending,
Poll::Ready(Ok(())) => { Poll::Ready(Ok(())) => {
if discard_buf.filled().is_empty() { if discard_buf.filled().is_empty() {
debug_assert_eq!(buf.filled().len(), original_filled);
Poll::Ready(Ok(())) Poll::Ready(Ok(()))
} else { } else {
Poll::Ready(Err(Error::other("input provided more bytes than specified"))) Poll::Ready(Err(Error::other("input provided more bytes than specified")))
@@ -65,46 +58,30 @@ where
Poll::Ready(Err(err)) => Poll::Ready(Err(err)), Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
}; };
} }
// Save the initial length
let before = original_filled;
let remaining = match usize::try_from(*this.remaining) { // Poll the inner reader
Ok(remaining) => remaining, let this = self.as_mut().project();
Err(_) => usize::MAX, let poll = this.inner.poll_read(cx, buf);
};
let allowed = remaining.min(buf.remaining()); if let Poll::Ready(Ok(())) = &poll {
let read = if allowed == buf.remaining() { let after = buf.filled().len();
let before = buf.filled().len(); let read = (after - before) as i64;
match this.inner.as_mut().poll_read(cx, buf) { if read == 0 && *this.remaining > 0 {
Poll::Pending => return Poll::Pending, return Poll::Ready(Err(Error::new(
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), std::io::ErrorKind::UnexpectedEof,
Poll::Ready(Ok(())) => buf.filled().len() - before, IncompleteBody {
remaining: *this.remaining,
},
)));
} }
} else { *this.remaining -= read;
this.scratch.resize(allowed, 0); if *this.remaining < 0 {
let mut scratch_buf = ReadBuf::new(&mut this.scratch[..allowed]); return Poll::Ready(Err(Error::other("input provided more bytes than specified")));
match this.inner.as_mut().poll_read(cx, &mut scratch_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
Poll::Ready(Ok(())) => {
let read = scratch_buf.filled().len();
buf.put_slice(scratch_buf.filled());
read
}
} }
};
if read == 0 {
return Poll::Ready(Err(Error::new(
std::io::ErrorKind::UnexpectedEof,
IncompleteBody {
remaining: *this.remaining,
},
)));
} }
let read = match i64::try_from(read) { poll
Ok(read) => read,
Err(_) => return Poll::Ready(Err(Error::other("read count exceeds i64::MAX"))),
};
*this.remaining -= read;
Poll::Ready(Ok(()))
} }
} }
@@ -163,12 +140,7 @@ mod tests {
assert!(err.is_some()); assert!(err.is_some());
let err = err.unwrap(); let err = err.unwrap();
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(
err.get_ref()
.and_then(|source| source.downcast_ref::<std::io::Error>())
.is_some_and(|source| source.to_string().contains("more bytes than specified"))
);
} }
#[tokio::test] #[tokio::test]
@@ -183,17 +155,6 @@ mod tests {
assert_eq!(&buf, data); assert_eq!(&buf, data);
} }
#[tokio::test]
async fn test_hardlimit_reader_zero_capacity_read_does_not_consume_input() {
let mut reader = HardLimitReader::new(BufReader::new(&b"abc"[..]), 3);
let mut empty = [];
assert_eq!(reader.read(&mut empty).await.expect("zero-capacity read should succeed"), 0);
let mut out = Vec::new();
reader.read_to_end(&mut out).await.expect("input should remain readable");
assert_eq!(out, b"abc");
}
#[tokio::test] #[tokio::test]
async fn test_hardlimit_reader_short_input_returns_unexpected_eof() { async fn test_hardlimit_reader_short_input_returns_unexpected_eof() {
let data = b"abc"; let data = b"abc";
@@ -234,18 +195,4 @@ mod tests {
assert_eq!(err.kind(), std::io::ErrorKind::Other); assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(err.to_string().contains("more bytes than specified")); assert!(err.to_string().contains("more bytes than specified"));
} }
#[tokio::test]
async fn test_hardlimit_reader_caps_each_read_before_reporting_extra_bytes() {
let mut reader = HardLimitReader::new(BufReader::new(&b"abcdef"[..]), 3);
let mut out = Vec::new();
let err = reader
.read_to_end(&mut out)
.await
.expect_err("bytes beyond the declared limit must be rejected");
assert_eq!(out, b"abc");
assert!(err.to_string().contains("more bytes than specified"));
}
} }
+59 -3
View File
@@ -715,6 +715,13 @@ async fn get_http_client(url: &str) -> io::Result<Client> {
Ok(cached.client_for(disable_proxy)) Ok(cached.client_for(disable_proxy))
} }
async fn get_fresh_http_client(url: &str) -> io::Result<Client> {
let tuning = internode_http_client_tuning();
let disable_proxy = should_disable_proxy_for_url(url, tuning);
let outbound_tls = crate::http_runtime_sources::outbound_tls_state().await;
build_http_client(disable_proxy, tuning, &outbound_tls).await
}
fn internode_request_context(method: &Method, url: &str, operation: Option<&'static str>) -> InternodeHttpRequestContext { fn internode_request_context(method: &Method, url: &str, operation: Option<&'static str>) -> InternodeHttpRequestContext {
let target = reqwest::Url::parse(url) let target = reqwest::Url::parse(url)
.ok() .ok()
@@ -962,6 +969,28 @@ impl HttpReader {
Self::with_capacity_and_stall_timeout(url, method, headers, body, 0, stall_timeout).await Self::with_capacity_and_stall_timeout(url, method, headers, body, 0, stall_timeout).await
} }
pub async fn new_fresh_connection_with_stall_timeout(
url: String,
method: Method,
headers: HeaderMap,
body: Option<Vec<u8>>,
stall_timeout: Option<Duration>,
) -> io::Result<Self> {
let init = Self::open(&url, &method, &headers, body, stall_timeout, true).await?;
Ok(Self {
inner: StreamReader::new(init.stream),
url,
method,
headers,
track_internode_metrics: init.track_internode_metrics,
internode_operation: init.internode_operation,
stall_timer: None,
stall_timeout: init.stall_timeout,
request_started: init.request_started,
duration_recorded: false,
})
}
/// Create a new HttpReader from a URL. The request is performed immediately. /// Create a new HttpReader from a URL. The request is performed immediately.
pub async fn with_capacity( pub async fn with_capacity(
url: String, url: String,
@@ -981,7 +1010,7 @@ impl HttpReader {
_read_buf_size: usize, _read_buf_size: usize,
stall_timeout: Option<Duration>, stall_timeout: Option<Duration>,
) -> io::Result<Self> { ) -> io::Result<Self> {
let init = Self::open(&url, &method, &headers, body, stall_timeout).await?; let init = Self::open(&url, &method, &headers, body, stall_timeout, false).await?;
Ok(Self { Ok(Self {
inner: StreamReader::new(init.stream), inner: StreamReader::new(init.stream),
url, url,
@@ -1002,10 +1031,16 @@ impl HttpReader {
headers: &HeaderMap, headers: &HeaderMap,
body: Option<Vec<u8>>, body: Option<Vec<u8>>,
stall_timeout: Option<Duration>, stall_timeout: Option<Duration>,
force_fresh_connection: bool,
) -> io::Result<HttpReaderInit> { ) -> io::Result<HttpReaderInit> {
let track_internode_metrics = is_internode_rpc_url(url); let track_internode_metrics = is_internode_rpc_url(url);
let internode_operation = internode_rpc_operation(url); let internode_operation = internode_rpc_operation(url);
let client = get_http_client(url).await.inspect_err(|_| { let client = if force_fresh_connection {
get_fresh_http_client(url).await
} else {
get_http_client(url).await
}
.inspect_err(|_| {
record_internode_error(track_internode_metrics, internode_operation); record_internode_error(track_internode_metrics, internode_operation);
})?; })?;
let mut request: RequestBuilder = client.request(method.clone(), url).headers(headers.clone()); let mut request: RequestBuilder = client.request(method.clone(), url).headers(headers.clone());
@@ -1121,7 +1156,28 @@ impl HttpChunkReader {
body: Option<Vec<u8>>, body: Option<Vec<u8>>,
stall_timeout: Option<Duration>, stall_timeout: Option<Duration>,
) -> io::Result<Self> { ) -> io::Result<Self> {
let init = HttpReader::open(&url, &method, &headers, body, stall_timeout).await?; let init = HttpReader::open(&url, &method, &headers, body, stall_timeout, false).await?;
Ok(Self {
inner: init.stream,
current: None,
track_internode_metrics: init.track_internode_metrics,
internode_operation: init.internode_operation,
stall_timer: None,
stall_timeout: init.stall_timeout,
request_started: init.request_started,
duration_recorded: false,
consecutive_empty_chunks: 0,
})
}
pub async fn new_fresh_connection_with_stall_timeout(
url: String,
method: Method,
headers: HeaderMap,
body: Option<Vec<u8>>,
stall_timeout: Option<Duration>,
) -> io::Result<Self> {
let init = HttpReader::open(&url, &method, &headers, body, stall_timeout, true).await?;
Ok(Self { Ok(Self {
inner: init.stream, inner: init.stream,
current: None, current: None,
+1 -1
View File
@@ -102,7 +102,7 @@ bytes.workspace = true
hex-simd.workspace = true hex-simd.workspace = true
[dev-dependencies] [dev-dependencies]
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] } tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
serial_test = { workspace = true } serial_test = { workspace = true }
temp-env = { workspace = true } temp-env = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
+12 -111
View File
@@ -65,7 +65,6 @@ const LOG_SUBSYSTEM_FOLDER: &str = "folder";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const LOG_SUBSYSTEM_HEAL: &str = "heal"; const LOG_SUBSYSTEM_HEAL: &str = "heal";
const EVENT_SCANNER_FOLDER_STATE: &str = "scanner_folder_state"; const EVENT_SCANNER_FOLDER_STATE: &str = "scanner_folder_state";
const EVENT_SCANNER_METADATA_CORRUPT: &str = "scanner_metadata_corrupt";
const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action"; const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action";
const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission"; const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission";
const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state"; const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state";
@@ -2155,34 +2154,17 @@ impl FolderScanner {
self.record_failed(&item.path); self.record_failed(&item.path);
if should_log_failed_object(into.failed_objects) { if should_log_failed_object(into.failed_objects) {
if let GetSizeFailureAction::HealMetadata { object } = &failure_action { warn!(
error!( target: "rustfs::scanner::folder",
target: "rustfs::scanner::folder", event = EVENT_SCANNER_FOLDER_STATE,
event = EVENT_SCANNER_METADATA_CORRUPT, component = LOG_COMPONENT_SCANNER,
component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER,
subsystem = LOG_SUBSYSTEM_FOLDER, path = %item.path,
drive = %self.local_disk.path().display(), failed_objects = into.failed_objects,
bucket = %item.bucket, state = "get_size_failed",
object = %object, error = %e,
metadata_path = %item.path, "Scanner folder failed to get object size"
failed_objects = into.failed_objects, );
state = "metadata_corrupt",
error = %e,
"Scanner detected corrupt object metadata"
);
} else {
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
path = %item.path,
failed_objects = into.failed_objects,
state = "get_size_failed",
error = %e,
"Scanner folder failed to get object size"
);
}
} }
} }
@@ -3072,59 +3054,12 @@ mod tests {
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass}; use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
use rustfs_filemeta::{FileInfo, FileMeta}; use rustfs_filemeta::{FileInfo, FileMeta};
use serial_test::serial; use serial_test::serial;
use std::io::Write;
#[cfg(unix)] #[cfg(unix)]
use std::os::unix::fs::{PermissionsExt, symlink}; use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use temp_env::{with_var, with_var_unset}; use temp_env::{with_var, with_var_unset};
use tracing_subscriber::fmt::MakeWriter;
use uuid::Uuid; use uuid::Uuid;
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
}
struct CapturedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl CapturedLogs {
fn contents(&self) -> String {
let buffer = self
.buffer
.lock()
.expect("captured logs mutex should not be poisoned")
.clone();
String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
}
}
impl Write for CapturedLogWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.buffer
.lock()
.expect("captured logs mutex should not be poisoned")
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for CapturedLogs {
type Writer = CapturedLogWriter;
fn make_writer(&'a self) -> Self::Writer {
CapturedLogWriter {
buffer: Arc::clone(&self.buffer),
}
}
}
#[test] #[test]
fn scanner_size_summary_application_saturates_usage_counters() { fn scanner_size_summary_application_saturates_usage_counters() {
let target = "arn:minio:replication::target".to_string(); let target = "arn:minio:replication::target".to_string();
@@ -4607,19 +4542,9 @@ mod tests {
assert!(budget.entries_visited() >= 1); assert!(budget.entries_visited() >= 1);
} }
#[tokio::test(flavor = "current_thread")] #[tokio::test]
#[serial] #[serial]
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() { async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.json()
.with_max_level(tracing::Level::ERROR)
.with_writer(logs.clone())
.with_ansi(false)
.without_time()
.finish();
let _subscriber_guard = tracing::subscriber::set_default(subscriber);
let (mut scanner, temp_dir) = build_test_scanner().await; let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
@@ -4671,30 +4596,6 @@ mod tests {
assert!(!budget.budget_elapsed()); assert!(!budget.budget_elapsed());
assert_eq!(budget.reason(), None); assert_eq!(budget.reason(), None);
let captured = logs.contents();
assert!(
!captured.contains("failed to check XL2 v1 format"),
"the context-free filemeta parser error must not be emitted"
);
let events = captured
.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("captured scanner log should be valid JSON"))
.filter(|line| line["fields"]["event"] == EVENT_SCANNER_METADATA_CORRUPT)
.collect::<Vec<_>>();
assert_eq!(
events.len(),
1,
"one corrupt metadata observation must emit one scanner-owned diagnostic event"
);
let fields = &events[0]["fields"];
assert_eq!(fields["component"], LOG_COMPONENT_SCANNER);
assert_eq!(fields["subsystem"], LOG_SUBSYSTEM_FOLDER);
assert_eq!(fields["drive"], temp_dir.to_string_lossy().as_ref());
assert_eq!(fields["bucket"], "bucket");
assert_eq!(fields["object"], "object");
assert_eq!(fields["metadata_path"], metadata_path.to_string_lossy().as_ref());
assert_eq!(fields["state"], "metadata_corrupt");
let retry_budget = ScannerCycleBudget::new_with_progress_tracking( let retry_budget = ScannerCycleBudget::new_with_progress_tracking(
&parent, &parent,
crate::scanner_budget::ScannerCycleBudgetConfig { crate::scanner_budget::ScannerCycleBudgetConfig {
+15 -9
View File
@@ -3849,6 +3849,17 @@ impl ScannerIODisk for Disk {
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) { let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions, Ok(versions) => versions,
Err(e) => { Err(e) => {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %item.bucket,
object = %item.object_path(),
state = "file_info_versions_failed",
error = %e,
"Scanner disk bucket failed to resolve file info versions"
);
return Err(scanner_metadata_corrupt_error( return Err(scanner_metadata_corrupt_error(
format!("failed to resolve file info versions: {e}"), format!("failed to resolve file info versions: {e}"),
&item.bucket, &item.bucket,
@@ -4012,7 +4023,7 @@ mod tests {
use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem; use crate::scanner_folder::ScannerItem;
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats}; use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
use crate::{ use crate::{
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
@@ -4237,21 +4248,16 @@ mod tests {
async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await; let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple()); let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple());
store store.pools[1].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default()) .make_bucket(&bucket, &MakeBucketOptions::default())
.await .await
.expect("bucket and its authoritative metadata should be created"); .expect("bucket should be created only in the second pool");
let body = b"second-only"; let body = b"second-only";
let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); let mut reader = ScannerPutObjReader::from_vec(body.to_vec());
store.pools[1] store.pools[1].disk_set[0]
.put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default()) .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default())
.await .await
.expect("object should be written only to the second pool"); .expect("object should be written only to the second pool");
store.pools[0]
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("bucket should be removed from the first pool only");
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
let ctx = CancellationToken::new(); let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
+1 -1
View File
@@ -278,7 +278,7 @@ pub(crate) mod scan {
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
}; };
#[cfg(test)] #[cfg(test)]
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO}; pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO};
} }
pub(crate) mod scanner_io { pub(crate) mod scanner_io {
+3
View File
@@ -659,6 +659,9 @@ mod test {
// Port should be in valid range (u16 max is always <= 65535) // Port should be in valid range (u16 max is always <= 65535)
assert!(port1 > 0); assert!(port1 > 0);
assert!(port2 > 0); assert!(port2 > 0);
// Different calls should typically return different ports
assert_ne!(port1, port2);
} }
#[test] #[test]
+1 -1
View File
@@ -16,6 +16,7 @@ for later deletion.
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract. - `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
- `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus. - `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus.
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
- `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array.
@@ -33,7 +34,6 @@ for later deletion.
- `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later. - `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later.
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
- `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor.
## Review Checklist ## Review Checklist
+1 -1
View File
@@ -111,7 +111,7 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and
| `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. |
| `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | | `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. |
| `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | | `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. |
| `SITE_REPLICATION_PEER_CLIENT` | `rustfs/src/admin/handlers/site_replication.rs` | Process-global owner-local cache | Site-replication peer client cache stays private to site-replication handlers. The state RMW transaction holds no process-local mutex — see `rustfs/src/admin/site_replication_state.rs`. | | `SITE_REPLICATION_PEER_CLIENT`, `SITE_REPLICATION_STATE_LOCK` | `rustfs/src/admin/handlers/site_replication.rs` | Process-global owner-local cache / guard | Site-replication peer client cache and state lock stay private to site-replication handlers. |
| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED`, `PERSISTED_NOTIFY_MODULE_ENABLED`, `PERSISTED_AUDIT_MODULE_ENABLED`, `PERSISTED_MODULE_SWITCH_CONFIGURED` | `rustfs/src/server/audit.rs`, `rustfs/src/server/event.rs`, `rustfs/src/server/module_switch.rs` | Process-global owner-local toggles | Audit/notify module snapshots stay private to the server module switch owners. | | `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED`, `PERSISTED_NOTIFY_MODULE_ENABLED`, `PERSISTED_AUDIT_MODULE_ENABLED`, `PERSISTED_MODULE_SWITCH_CONFIGURED` | `rustfs/src/server/audit.rs`, `rustfs/src/server/event.rs`, `rustfs/src/server/module_switch.rs` | Process-global owner-local toggles | Audit/notify module snapshots stay private to the server module switch owners. |
| `DELETE_TAIL_TOTAL`, `DELETE_CLEANUP_TOTAL`, `DELETE_REPLICATION_TOTAL`, `DELETE_NOTIFY_TOTAL` | `rustfs/src/delete_tail_activity.rs` | Process-global owner-local counters | Delete-tail activity counters stay private behind delete-tail activity helpers. | | `DELETE_TAIL_TOTAL`, `DELETE_CLEANUP_TOTAL`, `DELETE_REPLICATION_TOTAL`, `DELETE_NOTIFY_TOTAL` | `rustfs/src/delete_tail_activity.rs` | Process-global owner-local counters | Delete-tail activity counters stay private behind delete-tail activity helpers. |
| `EMBEDDED_SERVER_STARTED` | `rustfs/src/startup_lifecycle.rs` | Process-global owner-local guard | Embedded startup single-start protection stays private to startup lifecycle. | | `EMBEDDED_SERVER_STARTED` | `rustfs/src/startup_lifecycle.rs` | Process-global owner-local guard | Embedded startup single-start protection stays private to startup lifecycle. |
@@ -1,218 +0,0 @@
# Object I/O (GET/PUT) tuning A/B matrix runbook
> Scope: **parameter tuning** — measuring the effect of changing one
> `RUSTFS_*` runtime knob at a time, against a fixed binary. This is
> deliberately different from the code-change A/B gate in
> [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) and the formal
> ABBA validation in
> [`hotpath-warp-abba-runbook.md`](hotpath-warp-abba-runbook.md), which compare
> a baseline binary against a candidate binary.
>
> When a knob change turns out to need a code change, use those two runbooks
> for the code-level validation and come back here for the knob-level sweep.
## 1. What this runbook answers
For each tuning knob it answers three questions:
1. Which stage is actually slow — `set_disk_encode`, `set_disk_rename`,
`metadata_fanout`, `bitrot_verify`, etc.?
2. Is the knob the real bottleneck, or is the stage slow for another reason?
3. Does widening/loosening the knob buy throughput without an unacceptable
memory (RSS) or tail-latency regression?
The core discipline is **one variable per A/B cell**. Never change two knobs in
the same cell, or the result is unexplainable.
## 2. Prerequisites
- Linux bench host (or an ansible-managed cluster); a laptop smoke run is too
noisy to decide anything.
- `warp` on `PATH` (or pass `--warp-bin` to the driver).
- The observability metrics runtime **enabled**. The stage histograms below are
not emitted when `RUSTFS_OBS_METRICS_EXPORT_ENABLED=false` or the runtime is
otherwise off — see
[`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) for the no-log /
no-monitor baseline env.
- A warm, disposable data set. Recreate the bucket per run; do not bench against
production data.
Load driver and gate are reused, not reimplemented:
- `scripts/run_object_batch_bench_enhanced.sh` — warp driver with rounds,
median aggregation, `baseline_compare.csv`, and Prometheus service-metric
capture.
- `scripts/hotpath_warp_ab_gate.sh` — relative budget gate over the deltas.
- `scripts/run_hotpath_warp_ab.sh` — optional orchestrator when a knob needs
the full baseline-vs-candidate treatment (e.g. two different defaults).
## 3. Fixed test conditions (lock before you start)
Record these per run; a result without them is not reproducible:
```text
nodes, disks_per_node, total_disks, cpu_per_node, mem_per_node,
network, erasure_set_drive_count, endpoint_mode (direct|lb),
rustfs_commit_sha, warp --version, durability mode
```
Workload matrix (the same shapes the hotpath gate uses, expanded for the
stage-breakdown object sizes):
| Workload | mode | sizes |
| --- | --- | --- |
| small-fixed | put / get | 4KiB, 100KiB |
| ec-boundary | put / get | 1MiB, 4MiB |
| large-stream | put / get | 10MiB, 16MiB, 32MiB |
| mixed | mixed | 256KiB |
Concurrency ladder: `8, 16, 32, 64` (add `96, 128` on a bigger rig). Duration
`120s`, `--rounds >= 3`, cooldown `>= 30s`.
Isolate background noise before the sweep: scanner deep-verify, heal,
replication, lifecycle transition, periodic capacity refresh — record whether
each is on rather than silently assuming it is off.
## 4. Measurement stack
The code already instruments every stage below. Drive each A/B cell with these
histograms (names verified against `crates/io-metrics/src/lib.rs`):
- PUT stages: `rustfs_s3_put_object_stage_duration_ms{stage=...}` — compute
P50/P95/P99 per stage. Stages: `app_bucket_validate`, `app_sse_config_lookup`,
`app_object_lock_config_lookup`, `app_put_opts_build`, `app_prelookup`,
`ingress_prepare`, `app_encryption_prepare`, `app_replication_decision`,
`app_store_put`, `app_post_store_bookkeeping`, `app_capacity_update`,
`set_disk_writer_setup`, `set_disk_encode`, `set_disk_rename`,
`set_disk_old_data_cleanup`.
- GET stages: `rustfs_io_get_object_stage_duration_seconds{path=..., stage=...}`
the `path` label separates the read paths: `legacy_duplex`, `codec_streaming`,
`direct_memory`, `body_cache`, `inline_direct`, `internal_meta`,
`remote_transition`, `set_disk`, `empty`. Stages: `metadata`,
`metadata_cache_lookup`, `metadata_fanout`, `metadata_resolve`, `object_info`,
`path_decision`, `quorum_reached`, `range`, `reader_setup`,
`stripe_read`, `stripe_read_first_shard`, `stripe_read_quorum`, `decode`,
`reconstruct`, `emit`, `fill`, `output_poll`, `output_lock_wait`,
`bitrot_verify`, `first_byte`, `full_body`, `response_handoff`,
`lock_acquire`.
- EC memory pressure: `rustfs_ec_encode_inflight_bytes_current` and the
allocator reclaim gauge; plus node RSS and CPU.
Host telemetry (collect alongside every cell):
```bash
pidstat -durh 5 > telemetry/pidstat.txt &
mpstat 5 > telemetry/mpstat.txt &
iostat -xz 5 > telemetry/iostat.txt &
```
## 5. Tuning knob catalog
Defaults are verified against `crates/config/src/constants/object.rs` and
`crates/ecstore/src/erasure/coding/encode.rs`.
### 5.1 PUT
| Knob | Default | Controls | Validating stage | Risk if widened |
| --- | --- | --- | --- | --- |
| `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` | 32MiB | EC encode producer/consumer memory budget (blocks queued between encode and shard write) | `set_disk_encode` P95 + `rustfs_ec_encode_inflight_bytes_current` | RSS growth under high concurrency |
| `RUSTFS_OBJECT_IO_BUFFER_SIZE` | 128KiB | Streaming read-in / write-out block size | `ingress_prepare`, `set_disk_encode` | Larger buffers = fewer polls, more resident memory |
| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | duplex pipe capacity (shared, but PUT path uses it less than GET) | `set_disk_encode` feed smoothness | Memory per in-flight request |
| `RUSTFS_DURABILITY_MODE` / `RUSTFS_DRIVE_SYNC_ENABLE` | mode-dependent | per-shard fsync/sync discipline on commit | `set_disk_rename` P99 | Weakening it changes the durability contract — treat as a deliberate tradeoff, not a free win |
| `RUSTFS_RUNTIME_WORKER_THREADS` / `RUSTFS_RUNTIME_MAX_BLOCKING_THREADS` | Tokio defaults | async workers + `spawn_blocking` pool feeding per-block encode | `set_disk_encode` P95 + mpstat | Oversubscription |
### 5.2 GET
| Knob | Default | Controls | Validating stage | Risk if enabled |
| --- | --- | --- | --- | --- |
| `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` | `off` | switches the read path from `legacy_duplex` to the pull-based `ErasureDecodeReader` (`codec_streaming`) | compare `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` | behavioral change to the read path; rollout is `off` by default for a reason |
| `RUSTFS_GET_CODEC_STREAMING_ENGINE` | `legacy` | `legacy` vs `rustfs` decode engine under the streaming reader | `reconstruct`/`decode` per `path` | engine swap on a correctness-critical path |
| `RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE` | `false` | multipart objects on the streaming reader | same, multipart cells | wider format coverage |
| `RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE` (+ `_MAX_SIZE`, `_FIRST_READER_SETUP`) | `false` / 512KiB | prefer data-shard readers before parity | `stripe_read_first_shard`/`stripe_read_quorum` | shard-selection order change |
| `RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY` | `false` | skip per-shard HighwayHash verify | `bitrot_verify` | **do not default on** — measures the theoretical ceiling only |
| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | legacy GET in-process pipe capacity | `output_lock_wait`/`output_poll` | memory per in-flight GET |
| `RUSTFS_GET_SEEK_BUFFER_ENABLE` | `false` | in-memory seek buffer for small GET | `first_byte` | experimental, startup-latched — see [`get-path-experimental-switches.md`](get-path-experimental-switches.md) |
| `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` | `false` | adds `response_handoff` attribution (metrics only) | `response_handoff` | small per-request bookkeeping cost |
## 6. A/B matrix
Run each row as an independent cell. Baseline is the shipped default; candidate
is one knob moved. Everything else (topology, sizes, concurrency, rounds,
durability) stays fixed.
### 6.1 PUT
| # | Knob | Baseline | Candidates | Signal metric | Judgement |
| --- | --- | --- | --- | --- | --- |
| P1 | encode in-flight | 32MiB | 48MiB, 64MiB, 96MiB | `set_disk_encode` P95 + throughput + RSS | throughput up AND `set_disk_encode` down AND RSS tolerable → budget was the binding constraint |
| P2 | object I/O buffer | 128KiB | 256KiB, 512KiB, 1MiB | `ingress_prepare` + `set_disk_encode` | encode smooths with bounded RSS → upstream feed was too small |
| P3 | duplex buffer | 4MiB | 8MiB, 16MiB | `set_disk_encode` feed variance | lower priority than P1/P2 |
| P4 | blocking threads | default | 512, 768, 1024 | `set_disk_encode` P95 + mpstat | per-block `spawn_blocking` is scheduler-bound if P95 falls |
| P5 | durability | current mode | weaker/stronger mode | `set_disk_rename` P99 | only as a **deliberate durability tradeoff**, never a silent default change |
| P6 | set drive count | current | other valid widths | all PUT stages | highest cost — only after P1P5, and only on a rebuildable topology |
### 6.2 GET
| # | Knob | Baseline | Candidates | Signal metric | Judgement |
| --- | --- | --- | --- | --- | --- |
| G1 | codec streaming rollout | `off` | `on` (pct ramp 10/50/100) | `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` + throughput | streaming beats duplex on `decode`+`output_lock_wait` and byte-for-byte output matches → candidate for default |
| G2 | codec streaming engine | `legacy` | `rustfs` | `reconstruct`/`decode` per `path` | engine swap is neutral-or-better on CPU with identical bytes |
| G3 | multipart streaming | `false` | `true` | multipart GET cells | only after G1 is stable on single-part |
| G4 | data-blocks-first | `false` | `true` | `stripe_read_first_shard`/`stripe_read_quorum` | fewer shard reads without a correctness regression |
| G5 | duplex buffer | 4MiB | 8MiB, 16MiB | `output_lock_wait`/`output_poll` (legacy path) | only if still on `legacy_duplex` |
| G6 | skip bitrot verify | `false` | `true` | `bitrot_verify` | **ceiling measurement only**; do not carry into production |
## 7. Execution sequence
1. Freeze the conditions in §3 and record the provenance block.
2. Run the baseline cell (all defaults) and capture stage histograms + host
telemetry.
3. Pick the **one** most-likely knob from the analysis. For large-object PUT
that is almost always `set_disk_encode` → P1; for GET it is G1 (the
`legacy_duplex``codec_streaming` switch).
4. Sweep that knob's candidate column one value at a time, same workload.
5. Read the decision table in §8; if the stage did not move, the knob is not
the bottleneck — stop widening it and pick the next stage.
Driver invocation for one PUT cell:
```bash
scripts/run_object_batch_bench_enhanced.sh \
--tool warp --endpoint http://127.0.0.1:9000 \
--access-key "$RUSTFS_ACCESS_KEY" --secret-key "$RUSTFS_SECRET_KEY" \
--bucket rustfs-put-tuning --warp-mode put \
--sizes 16MiB,32MiB --concurrency 32 --duration 120s --rounds 3 \
--out-dir target/bench/put-tuning-p1-64mib
```
## 8. Interpretation / decision table
| Stage high | Most likely cause | Next action |
| --- | --- | --- |
| `set_disk_encode` | per-block EC encode scheduling + in-flight budget | P1 → P4 → P2, in that order |
| `set_disk_rename` | commit tail (rename fan-out / RPC / fsync) | P5 (durability) and cluster tail analysis, not encode |
| `set_disk_writer_setup` | per-disk `BitrotWriter` + temp-file create | disk/filesystem metadata; per-disk fan-out cost |
| `set_disk_old_data_cleanup` | overwrite / versioned-object directory delete | confirm overwrite-vs-new-write; defer cleanup further |
| `metadata_fanout` / `metadata_resolve` | cross-disk `xl.meta` read + quorum | metadata cache hit rate; small-object fixed cost |
| `bitrot_verify` | HighwayHash verify on the read path | G6 ceiling only; do not default on |
| `output_lock_wait` / `output_poll` | legacy duplex backpressure | G1 (move off duplex) or G5 |
| `stripe_read*` | shard concurrency / selection | G4 shard-selection, disk/network tail |
## 9. Guardrails
- **Never weaken correctness for throughput**: read/write quorum, bitrot verify,
`xl.meta` validation, and durability (`RUSTFS_DURABILITY_MODE`) are integrity
contracts, not knobs. P5 and G6 are ceiling measurements and must be labelled
as such; do not carry their values into production without an explicit
durability/correctness decision.
- **One variable per cell.** A cell that changes two knobs is thrown away.
- **Memory is part of the result.** A throughput win with unbounded RSS growth
is a regression; record RSS and the EC in-flight gauge for every PUT cell.
- **Startup-latched knobs** (`RUSTFS_GET_SEEK_BUFFER_ENABLE`,
`RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE`, and the codec-streaming
switches) require a process restart to change — see
[`get-path-experimental-switches.md`](get-path-experimental-switches.md).
- **Archive the raw data.** Keep the `baseline_compare.csv`, `median_summary.csv`,
stage histograms, and host telemetry per cell; the conclusion must trace back
to them. Do not commit benchmark result snapshots to the repo — record them in
the issue tracker.
+1 -1
View File
@@ -59,7 +59,7 @@
{ {
default = rustPlatform.buildRustPackage { default = rustPlatform.buildRustPackage {
pname = "rustfs"; pname = "rustfs";
version = "1.0.0-rc.2"; version = "1.0.0-rc.1";
src = ./.; src = ./.;
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: rustfs name: rustfs
description: RustFS helm chart to deploy RustFS on kubernetes cluster. description: RustFS helm chart to deploy RustFS on kubernetes cluster.
type: application type: application
version: "1.0.0-rc.2" version: "1.0.0-rc.1"
appVersion: "1.0.0-rc.2" appVersion: "1.0.0-rc.1"
home: https://rustfs.com home: https://rustfs.com
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
maintainers: maintainers:
+2 -5
View File
@@ -1,9 +1,9 @@
%global _enable_debug_packages 0 %global _enable_debug_packages 0
%global _empty_manifest_terminate_build 0 %global _empty_manifest_terminate_build 0
%global prerelease rc.2 %global prerelease rc.1
Name: rustfs Name: rustfs
Version: 1.0.0 Version: 1.0.0
Release: rc.2 Release: rc.1
Summary: High-performance distributed object storage for MinIO alternative Summary: High-performance distributed object storage for MinIO alternative
License: Apache-2.0 License: Apache-2.0
@@ -58,9 +58,6 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
%_bindir/rustfs %_bindir/rustfs
%changelog %changelog
* Fri Aug 14 2026 overtrue <anzhengchao@gmail.com>
- Update RPM package to RustFS 1.0.0-rc.2
* Sat Aug 08 2026 overtrue <anzhengchao@gmail.com> * Sat Aug 08 2026 overtrue <anzhengchao@gmail.com>
- Update RPM package to RustFS 1.0.0-rc.1 - Update RPM package to RustFS 1.0.0-rc.1
+20 -72
View File
@@ -465,16 +465,6 @@ impl Operation for ImportBucketMetadata {
file_contents.push((file_path, content)); file_contents.push((file_path, content));
} }
let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
let quota_fleet_proof =
if durable_quota_import {
Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| {
s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster")
})?)
} else {
None
};
// Extract bucket names // Extract bucket names
let mut bucket_names = Vec::new(); let mut bucket_names = Vec::new();
for (file_path, _) in &file_contents { for (file_path, _) in &file_contents {
@@ -717,6 +707,21 @@ impl Operation for ImportBucketMetadata {
} }
BUCKET_QUOTA_CONFIG_FILE => { BUCKET_QUOTA_CONFIG_FILE => {
if let Err(e) = serde_json::from_slice::<BucketQuota>(&content) {
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m, Some(m) => m,
None => continue, None => continue,
@@ -825,6 +830,10 @@ impl Operation for ImportBucketMetadata {
} }
} }
// Persist the assembled metadata to disk. Prior to this, the import only mutated the
// in-memory `bucket_metadatas` map and returned 200, silently dropping every imported
// config. `metadata_sys::update` loads the on-disk metadata, overwrites the given config
// field and saves it, preserving any configs not present in the import archive.
for (bucket_name, metadata) in &bucket_metadatas { for (bucket_name, metadata) in &bucket_metadatas {
for (config_file, data) in imported_configs_to_persist(metadata) { for (config_file, data) in imported_configs_to_persist(metadata) {
let site_replication_item = imported_config_to_site_replication_item(bucket_name, metadata, config_file, &data)?; let site_replication_item = imported_config_to_site_replication_item(bucket_name, metadata, config_file, &data)?;
@@ -833,21 +842,7 @@ impl Operation for ImportBucketMetadata {
} else { } else {
None None
}; };
let persist_result = if config_file == BUCKET_QUOTA_CONFIG_FILE { if let Err(e) = metadata_sys::update(bucket_name, config_file, data).await {
let quota: BucketQuota =
serde_json::from_slice(&data).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?;
if quota.uses_durable_reservations() {
let proof = quota_fleet_proof.as_ref().ok_or_else(|| {
s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster")
})?;
metadata_sys::update_quota_if_incarnation(bucket_name, data, metadata.bucket_incarnation_id, proof).await
} else {
metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await
}
} else {
metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await
};
if let Err(e) = persist_result {
warn!( warn!(
event = EVENT_ADMIN_BUCKET_META_STATE, event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
@@ -890,26 +885,6 @@ impl Operation for ImportBucketMetadata {
} }
} }
fn imported_quota_requires_fleet_proof(file_contents: &[(String, Vec<u8>)]) -> S3Result<bool> {
let mut durable = false;
for (file_path, content) in file_contents {
let mut parts = file_path.split(SLASH_SEPARATOR);
let Some(_bucket) = parts.next() else {
continue;
};
if parts.next() != Some(BUCKET_QUOTA_CONFIG_FILE) {
continue;
}
let quota: BucketQuota =
serde_json::from_slice(content).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?;
if quota.has_unsupported_reservation_protocol() {
return Err(s3_error!(InvalidRequest, "unsupported bucket quota reservation protocol"));
}
durable |= quota.uses_durable_reservations();
}
Ok(durable)
}
/// The `(config_file, data)` pairs to persist for an imported bucket's metadata: every non-empty /// The `(config_file, data)` pairs to persist for an imported bucket's metadata: every non-empty
/// config field keyed by its on-disk config-file name, as owned data ready for /// config field keyed by its on-disk config-file name, as owned data ready for
/// `metadata_sys::update`. Empty fields are skipped so an import never overwrites an existing /// `metadata_sys::update`. Empty fields are skipped so an import never overwrites an existing
@@ -1101,31 +1076,4 @@ mod import_persist_tests {
assert!(!has_site_replication_item); assert!(!has_site_replication_item);
} }
#[test]
fn quota_import_preflight_rejects_invalid_and_unknown_protocols() {
let missing_limit = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
br#"{"quota":0,"reservation_protocol":1}"#.to_vec(),
)];
assert!(imported_quota_requires_fleet_proof(&missing_limit).is_err());
let unknown_protocol = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
br#"{"quota":0,"reservation_protocol":2,"reservation_quota":1024}"#.to_vec(),
)];
assert!(imported_quota_requires_fleet_proof(&unknown_protocol).is_err());
}
#[test]
fn quota_import_preflight_requires_proof_only_for_durable_quota() {
let legacy = vec![(format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), br#"{"quota":1024}"#.to_vec())];
assert!(!imported_quota_requires_fleet_proof(&legacy).expect("legacy quota should remain compatible"));
let durable = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
serde_json::to_vec(&BucketQuota::new(Some(1024))).expect("durable quota should encode"),
)];
assert!(imported_quota_requires_fleet_proof(&durable).expect("durable quota should pass preflight"));
}
} }
+4 -25
View File
@@ -22,7 +22,6 @@ use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys};
use crate::admin::storage_api::bucket::quota::checker::QuotaChecker; use crate::admin::storage_api::bucket::quota::checker::QuotaChecker;
use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::error::ApiError;
use crate::server::ADMIN_PREFIX; use crate::server::ADMIN_PREFIX;
use hyper::{Method, StatusCode}; use hyper::{Method, StatusCode};
use matchit::Params; use matchit::Params;
@@ -294,36 +293,16 @@ impl Operation for SetBucketQuotaHandler {
return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG)); return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG));
} }
let fleet_proof = if request.quota.is_some() {
Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| {
S3Error::with_message(
s3s::S3ErrorCode::ServiceUnavailable,
"durable quota capability is not confirmed across the cluster".to_string(),
)
})?)
} else {
None
};
let quota = BucketQuota::new(request.quota); let quota = BucketQuota::new(request.quota);
let metadata_sys_lock = bucket_metadata_from_context() let metadata_sys_lock = bucket_metadata_from_context()
.ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?; .ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?;
let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
let updated_at = match fleet_proof.as_ref() { let updated_at = quota_checker
Some(fleet_proof) => { .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id)
quota_checker .await
.set_durable_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id, fleet_proof) .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?;
.await
}
None => {
quota_checker
.set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id)
.await
}
}
.map_err(ApiError::from)?;
if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta { if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta {
bucket: bucket.clone(), bucket: bucket.clone(),
File diff suppressed because it is too large Load Diff
+603 -86
View File
@@ -12,8 +12,8 @@ use datafusion::{
use std::sync::Arc; use std::sync::Arc;
use crate::table_catalog::test_support::{ use crate::table_catalog::test_support::{
TestCatalogObjectBackend as TestTableCatalogObjectBackend, TestCatalogObjectRecord, TestCatalogPublishPause, TestCatalogObjectBackend as TestTableCatalogObjectBackend, TestCatalogObjectRecord,
TestTableCatalogStore, manifest_avro_bytes as test_manifest_avro_bytes, manifest_avro_bytes as test_manifest_avro_bytes,
manifest_avro_bytes_with_nullable_sequences as test_manifest_avro_bytes_with_nullable_sequences, manifest_avro_bytes_with_nullable_sequences as test_manifest_avro_bytes_with_nullable_sequences,
manifest_list_avro_bytes as test_manifest_list_avro_bytes, manifest_list_avro_entries as test_manifest_list_avro_entries, manifest_list_avro_bytes as test_manifest_list_avro_bytes, manifest_list_avro_entries as test_manifest_list_avro_entries,
table_metadata_json as test_table_metadata_json, table_metadata_json as test_table_metadata_json,
@@ -6328,96 +6328,181 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() {
assert_eq!(unchanged.generation, current.generation); assert_eq!(unchanged.generation, current.generation);
} }
/// Table-driven fold of the three commit-rejection cases whose bodies were
/// identical apart from four literals (backlog#1837 PR3). Each row keeps its
/// original manifest-list sequence, data-file name, manifest-entry snapshot
/// id, and failure message, so no poison combination is lost.
#[tokio::test] #[tokio::test]
async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { async fn row_level_conflict_rejects_stale_new_manifest_sequence() {
// (case, manifest-list sequence, data-file suffix, manifest-entry snapshot id, expected failure) let store = TestTableCatalogStore::default();
let cases: &[(&str, i64, &str, i64, &str)] = &[ let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
( let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
"stale-new-manifest-sequence", let created = create_standard_events_table(&store, &metadata_backend, &namespace).await;
1, let table_location = created.metadata["location"]
"11", .as_str()
11, .expect("created metadata should have table location");
"new manifest sequence must match the committed snapshot", let current = store
), .load_table("warehouse", "analytics", "events")
( .await
"stale-added-entry-sequence", .expect("table lookup should succeed")
2, .expect("table should exist");
"11", let manifest_list = format!("{table_location}/metadata/snap-11.avro");
11, let manifest = format!("{table_location}/metadata/manifest-11.avro");
"added file sequence must match the new manifest", let data_file = format!("{table_location}/data/part-11.parquet");
), seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 11).await;
( seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await;
"historical-change-in-new-manifest", let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({
2, "updates": [
"10", {
10, "action": "add-snapshot",
"new manifest must not claim a historical changed entry", "snapshot": {
), "snapshot-id": 11,
]; "sequence-number": 2,
"timestamp-ms": 2234,
for (case, manifest_list_sequence, data_file_suffix, entry_snapshot_id, failure) in cases { "manifest-list": manifest_list,
let store = TestTableCatalogStore::default(); "summary": {
let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); "operation": "append"
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
let created = create_standard_events_table(&store, &metadata_backend, &namespace).await;
let table_location = created.metadata["location"]
.as_str()
.expect("created metadata should have table location");
let current = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should exist");
let manifest_list = format!("{table_location}/metadata/snap-11.avro");
let manifest = format!("{table_location}/metadata/manifest-11.avro");
let data_file = format!("{table_location}/data/part-{data_file_suffix}.parquet");
seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await;
seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, *entry_snapshot_id, 1)]).await;
let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({
"updates": [
{
"action": "add-snapshot",
"snapshot": {
"snapshot-id": 11,
"sequence-number": 2,
"timestamp-ms": 2234,
"manifest-list": manifest_list,
"summary": {
"operation": "append"
}
} }
} }
] }
})) ]
.expect("append request should parse"); }))
.expect("append request should parse");
let Err(error) = commit_table_response( let error = commit_table_response(
&store, &store,
&trusted_table_commit_backend(&metadata_backend), &trusted_table_commit_backend(&metadata_backend),
"warehouse", "warehouse",
&namespace, &namespace,
"events", "events",
append_request, append_request,
) )
.await
.expect_err("new manifest sequence must match the committed snapshot");
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest);
let unchanged = store
.load_table("warehouse", "analytics", "events")
.await .await
else { .expect("table lookup should succeed")
panic!("[{case}] {failure}"); .expect("table should still exist");
}; assert_eq!(unchanged.metadata_location, current.metadata_location);
assert_eq!(unchanged.version_token, current.version_token);
assert_eq!(unchanged.generation, current.generation);
}
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest, "[{case}] {failure}"); #[tokio::test]
let unchanged = store async fn row_level_conflict_rejects_stale_added_entry_sequence() {
.load_table("warehouse", "analytics", "events") let store = TestTableCatalogStore::default();
.await let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
.expect("table lookup should succeed") let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
.expect("table should still exist"); let created = create_standard_events_table(&store, &metadata_backend, &namespace).await;
assert_eq!(unchanged.metadata_location, current.metadata_location, "[{case}] {failure}"); let table_location = created.metadata["location"]
assert_eq!(unchanged.version_token, current.version_token, "[{case}] {failure}"); .as_str()
assert_eq!(unchanged.generation, current.generation, "[{case}] {failure}"); .expect("created metadata should have table location");
} let current = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should exist");
let manifest_list = format!("{table_location}/metadata/snap-11.avro");
let manifest = format!("{table_location}/metadata/manifest-11.avro");
let data_file = format!("{table_location}/data/part-11.parquet");
seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await;
seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await;
let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({
"updates": [
{
"action": "add-snapshot",
"snapshot": {
"snapshot-id": 11,
"sequence-number": 2,
"timestamp-ms": 2234,
"manifest-list": manifest_list,
"summary": {
"operation": "append"
}
}
}
]
}))
.expect("append request should parse");
let error = commit_table_response(
&store,
&trusted_table_commit_backend(&metadata_backend),
"warehouse",
&namespace,
"events",
append_request,
)
.await
.expect_err("added file sequence must match the new manifest");
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest);
let unchanged = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should still exist");
assert_eq!(unchanged.metadata_location, current.metadata_location);
assert_eq!(unchanged.version_token, current.version_token);
assert_eq!(unchanged.generation, current.generation);
}
#[tokio::test]
async fn row_level_conflict_rejects_historical_change_in_new_manifest() {
let store = TestTableCatalogStore::default();
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
let created = create_standard_events_table(&store, &metadata_backend, &namespace).await;
let table_location = created.metadata["location"]
.as_str()
.expect("created metadata should have table location");
let current = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should exist");
let manifest_list = format!("{table_location}/metadata/snap-11.avro");
let manifest = format!("{table_location}/metadata/manifest-11.avro");
let data_file = format!("{table_location}/data/part-10.parquet");
seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await;
seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await;
let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({
"updates": [
{
"action": "add-snapshot",
"snapshot": {
"snapshot-id": 11,
"sequence-number": 2,
"timestamp-ms": 2234,
"manifest-list": manifest_list,
"summary": {
"operation": "append"
}
}
}
]
}))
.expect("append request should parse");
let error = commit_table_response(
&store,
&trusted_table_commit_backend(&metadata_backend),
"warehouse",
&namespace,
"events",
append_request,
)
.await
.expect_err("new manifest must not claim a historical changed entry");
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest);
let unchanged = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should still exist");
assert_eq!(unchanged.metadata_location, current.metadata_location);
assert_eq!(unchanged.version_token, current.version_token);
assert_eq!(unchanged.generation, current.generation);
} }
#[tokio::test] #[tokio::test]
@@ -7837,6 +7922,34 @@ fn commit_table_request_uses_rest_commit_fields() {
assert_eq!(request.writer.as_deref(), Some("pyiceberg")); assert_eq!(request.writer.as_deref(), Some("pyiceberg"));
} }
#[derive(Default)]
struct TestTableCatalogStore {
table_buckets: tokio::sync::Mutex<Vec<crate::table_catalog::TableBucketEntry>>,
namespaces: tokio::sync::Mutex<Vec<crate::table_catalog::NamespaceEntry>>,
tables: tokio::sync::Mutex<Vec<crate::table_catalog::TableEntry>>,
views: tokio::sync::Mutex<Vec<crate::table_catalog::ViewEntry>>,
commits: tokio::sync::Mutex<Vec<crate::table_catalog::CommitLogEntry>>,
fail_put_table_bucket: tokio::sync::Mutex<bool>,
register_table_pause: Option<TestCatalogPublishPause>,
commit_table_pause: Option<TestCatalogPublishPause>,
}
#[derive(Clone, Default)]
struct TestCatalogPublishPause {
started: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
}
impl TestCatalogPublishPause {
async fn wait_started(&self) {
self.started.notified().await;
}
fn release(&self) {
self.release.notify_one();
}
}
fn trusted_table_commit_backend( fn trusted_table_commit_backend(
backend: &TestTableCatalogObjectBackend, backend: &TestTableCatalogObjectBackend,
) -> TableCommitObjectBackend<TestTableCatalogObjectBackend> { ) -> TableCommitObjectBackend<TestTableCatalogObjectBackend> {
@@ -8205,6 +8318,410 @@ async fn seed_object_table_for_metadata_maintenance(
.await; .await;
} }
#[async_trait::async_trait]
impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore {
async fn get_table_bucket(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::TableBucketEntry>> {
Ok(self
.table_buckets
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket)
.cloned())
}
async fn put_table_bucket(
&self,
entry: crate::table_catalog::TableBucketEntry,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await;
if *fail_put_table_bucket {
*fail_put_table_bucket = false;
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"injected table bucket write failure".to_string(),
));
}
drop(fail_put_table_bucket);
let mut table_buckets = self.table_buckets.lock().await;
table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket);
table_buckets.push(entry);
Ok(())
}
async fn create_namespace(
&self,
entry: crate::table_catalog::NamespaceEntry,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
self.namespaces.lock().await.push(entry);
Ok(())
}
async fn list_namespaces(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::NamespaceEntry>> {
Ok(self
.namespaces
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket)
.cloned()
.collect())
}
async fn get_namespace(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::NamespaceEntry>> {
Ok(self
.namespaces
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned())
}
async fn update_namespace_properties(
&self,
table_bucket: &str,
namespace: &str,
update: crate::table_catalog::NamespacePropertiesUpdate,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::NamespacePropertiesUpdateResult> {
let mut namespaces = self.namespaces.lock().await;
let entry = namespaces
.iter_mut()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.ok_or_else(|| {
crate::table_catalog::TableCatalogStoreError::NotFound(format!("namespace {table_bucket}/{namespace}"))
})?;
Ok(update.apply_to(entry))
}
async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.namespaces
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace));
Ok(())
}
async fn create_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
self.tables.lock().await.push(entry);
Ok(())
}
async fn register_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
if let Some(pause) = &self.register_table_pause {
pause.started.notify_one();
pause.release.notified().await;
}
self.tables.lock().await.push(entry);
Ok(())
}
async fn register_table_with_publication(
&self,
entry: crate::table_catalog::TableEntry,
publication: &(dyn crate::table_catalog::TableCommitPublication + Sync),
) -> crate::table_catalog::TableCatalogStoreResult<()> {
publication.begin_table_bucket(&entry.table_bucket).await?;
if !publication.holds_table_bucket(&entry.table_bucket) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table registration requires a table-bucket publication fence".to_string(),
));
}
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
publication
.prepare(&entry.table_bucket, &entry.namespace, &entry.table)
.await?;
if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table registration requires a table publication fence".to_string(),
));
}
self.register_table(entry).await
}
async fn list_tables(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned()
.collect())
}
async fn list_all_tables(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket)
.cloned()
.collect())
}
async fn load_table(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)
.cloned())
}
async fn commit_table(
&self,
request: crate::table_catalog::TableCommitRequest,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::TableCommitResult> {
let mut tables = self.tables.lock().await;
let Some(index) = tables.iter().position(|entry| {
entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table
}) else {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table {}/{}/{}",
request.table_bucket, request.namespace, request.table
)));
};
let current = tables[index].clone();
if current.version_token != request.expected_version_token {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current table version token does not match expected token".to_string(),
));
}
if current.metadata_location != request.expected_metadata_location {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current table metadata location does not match expected location".to_string(),
));
}
if let Some(pause) = &self.commit_table_pause {
pause.started.notify_one();
pause.release.notified().await;
}
let mut next = current.clone();
next.metadata_location = request.new_metadata_location.clone();
next.version_token = "token-committed".to_string();
next.generation = next.generation.saturating_add(1);
tables[index] = next.clone();
drop(tables);
let commit_log = crate::table_catalog::CommitLogEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
commit_id: request.commit_id,
idempotency_key: request.idempotency_key,
table_id: current.table_id,
operation: request.operation,
expected_version_token: request.expected_version_token,
new_version_token: next.version_token.clone(),
previous_metadata_location: request.expected_metadata_location,
new_metadata_location: request.new_metadata_location,
requirements: request.requirements,
status: crate::table_catalog::CommitLogStatus::Committed,
writer: request.writer,
created_at: None,
updated_at: None,
};
self.commits.lock().await.push(commit_log.clone());
Ok(crate::table_catalog::TableCommitResult { table: next, commit_log })
}
async fn commit_table_with_publication(
&self,
request: crate::table_catalog::TableCommitRequest,
publication: &(dyn crate::table_catalog::TableCommitPublication + Sync),
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::TableCommitResult> {
publication
.prepare(&request.table_bucket, &request.namespace, &request.table)
.await?;
if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table commit requires a table publication fence".to_string(),
));
}
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
self.commit_table(request).await
}
async fn drop_table(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.tables
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table));
Ok(())
}
async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
self.views.lock().await.push(entry);
Ok(())
}
async fn list_views(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::ViewEntry>> {
Ok(self
.views
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned()
.collect())
}
async fn load_view(
&self,
table_bucket: &str,
namespace: &str,
view: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::ViewEntry>> {
Ok(self
.views
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)
.cloned())
}
async fn replace_view(
&self,
request: crate::table_catalog::ViewCommitRequest,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::ViewCommitResult> {
let mut views = self.views.lock().await;
let Some(index) = views.iter().position(|entry| {
entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view
}) else {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"view {}/{}/{}",
request.table_bucket, request.namespace, request.view
)));
};
let current = views[index].clone();
if current.version_token != request.expected_version_token {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current view version token does not match expected token".to_string(),
));
}
if current.metadata_location != request.expected_metadata_location {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current view metadata location does not match expected location".to_string(),
));
}
let mut next = current;
next.metadata_location = request.new_metadata_location;
next.version_token = "token-view-committed".to_string();
next.generation = next.generation.saturating_add(1);
views[index] = next.clone();
Ok(crate::table_catalog::ViewCommitResult { view: next })
}
async fn drop_view(
&self,
table_bucket: &str,
namespace: &str,
view: &str,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.views
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view));
Ok(())
}
async fn get_commit_by_id(
&self,
_table_bucket: &str,
_table_id: &str,
_commit_id: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::CommitLogEntry>> {
Ok(None)
}
async fn get_commit_by_idempotency_key(
&self,
_table_bucket: &str,
_table_id: &str,
_idempotency_key: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::CommitLogEntry>> {
Ok(None)
}
}
#[tokio::test] #[tokio::test]
async fn ensure_table_bucket_entry_seeds_enabled_bucket_before_namespace_create() { async fn ensure_table_bucket_entry_seeds_enabled_bucket_before_namespace_create() {
let store = TestTableCatalogStore::default(); let store = TestTableCatalogStore::default();
+1 -1
View File
@@ -2901,7 +2901,7 @@ async fn handle_misc_extension_request(req: &mut S3Request<Body>, route: &MiscEx
MiscExtRoute::ObjectLambda { bucket, object } => { MiscExtRoute::ObjectLambda { bucket, object } => {
let get_req = build_object_lambda_get_request(req, bucket, object)?; let get_req = build_object_lambda_get_request(req, bucket, object)?;
let usecase = default_object_usecase(); let usecase = default_object_usecase();
let get_resp = usecase.execute_get_object(get_req).await?; let get_resp = Box::pin(usecase.execute_get_object(get_req)).await?;
invoke_object_lambda_target(req, bucket, object, get_resp).await invoke_object_lambda_target(req, bucket, object, get_resp).await
} }
MiscExtRoute::ListenNotification { bucket } => { MiscExtRoute::ListenNotification { bucket } => {
+49 -16
View File
@@ -18,21 +18,28 @@
//! `config/site-replication/state.json` is mutated by read-modify-write //! `config/site-replication/state.json` is mutated by read-modify-write
//! sequences spread over many call sites: admin handlers, the retry-event //! sequences spread over many call sites: admin handlers, the retry-event
//! writers on every hook broadcast path, and the service-side reload driven //! writers on every hook broadcast path, and the service-side reload driven
//! over node RPC. //! over node RPC. Historically only some of them held the process-local
//! mutex and none held a distributed lock across the whole RMW, so
//! concurrent writers overwrote each other (single-process for the unlocked
//! writers, cross-node for everyone).
//! //!
//! `with_site_replication_state_lock` is the single transaction boundary: //! `with_site_replication_state_lock` is the single transaction boundary:
//! it holds the distributed config-object write lock (the pattern proven by //! it holds the process-local mutex AND the distributed config-object write
//! the repair state, `update_site_replication_repair_state`) for the //! lock (the pattern proven by the repair state,
//! duration of the caller's closure. The object lock is the sole mechanism — //! `update_site_replication_repair_state`) for the duration of the caller's
//! it is the only thing that can serialize two nodes of the same site, so a //! closure. All IO inside the closure must use the `*_no_lock` config
//! process-local lock must never be reintroduced in front of it as if it //! helpers — the locked variants would self-deadlock on the same object
//! added protection. All IO inside the closure must use the `*_no_lock` //! lock. Do not perform peer network calls or take other config locks
//! config helpers — the locked variants would self-deadlock on the same
//! object lock. Do not perform peer network calls or take other config locks
//! inside the closure. //! inside the closure.
//! //!
//! Lock order: lifecycle -> bucket operation -> repair admission //! The process-local mutex is transitional: call sites still outside this
//! -> state object lock -> per-bucket metadata. //! primitive serialize against migrated ones through it. Once every RMW
//! call site goes through here (P1-15 PR2) it will be removed, leaving the
//! object lock as the only mechanism.
//!
//! Lock order (unchanged from the historical comment next to the mutex):
//! lifecycle -> bucket operation -> repair admission -> state (process
//! mutex, then state object lock) -> per-bucket metadata.
use crate::admin::storage_api::runtime::ECStore; use crate::admin::storage_api::runtime::ECStore;
use crate::admin::storage_api::s3::{S3Error, S3ErrorCode, S3Result}; use crate::admin::storage_api::s3::{S3Error, S3ErrorCode, S3Result};
@@ -46,8 +53,24 @@ use super::runtime_sources::current_object_store_handle;
/// byte-level tolerant reload on the service side. /// byte-level tolerant reload on the service side.
pub(crate) const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; pub(crate) const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
/// Transitional process-local mutex — see the module docs. Stays private to
/// this module (owner-local static, enforced by
/// `scripts/check_architecture_migration_rules.sh`); callers go through
/// [`site_replication_state_process_guard`].
static SITE_REPLICATION_STATE_LOCK: std::sync::LazyLock<tokio::sync::Mutex<()>> =
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
/// Owner helper for the transitional process mutex: the RMW call sites in
/// `handlers::site_replication` that PR2 has not migrated to
/// [`with_site_replication_state_lock`] yet hold this guard so they stay
/// mutually exclusive with the migrated ones. Removed together with the
/// mutex once every call site runs inside the transaction boundary.
pub(crate) async fn site_replication_state_process_guard() -> tokio::sync::MutexGuard<'static, ()> {
SITE_REPLICATION_STATE_LOCK.lock().await
}
/// Run `operation` under the site-replication state transaction boundary: /// Run `operation` under the site-replication state transaction boundary:
/// the distributed state-object write lock. /// process mutex first, then the distributed state-object write lock.
pub(crate) async fn with_site_replication_state_lock<T, F, Fut>(operation: F) -> S3Result<T> pub(crate) async fn with_site_replication_state_lock<T, F, Fut>(operation: F) -> S3Result<T>
where where
T: Send + 'static, T: Send + 'static,
@@ -60,11 +83,21 @@ where
/// Context-store variant for callers that resolve their store from an /// Context-store variant for callers that resolve their store from an
/// explicit [`AppContext`] (the service-side reload driven over node RPC). /// explicit [`AppContext`] (the service-side reload driven over node RPC).
///
/// This is the whole boundary: a state-object write lock serializes writers
/// in *different* processes, which is what two nodes of one site are and
/// what a process mutex could never cover.
pub(crate) async fn with_site_replication_state_lock_on<T, F, Fut>(store: Arc<ECStore>, operation: F) -> S3Result<T> pub(crate) async fn with_site_replication_state_lock_on<T, F, Fut>(store: Arc<ECStore>, operation: F) -> S3Result<T>
where
T: Send + 'static,
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
{
let _process_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
with_site_replication_state_object_lock(store, operation).await
}
/// The distributed half of the boundary on its own: the state-object write
/// lock, without the process mutex. This is the only thing that serializes
/// writers in *different* processes (the mutex cannot), so it is also what
/// the separate-nodes regression test drives.
pub(crate) async fn with_site_replication_state_object_lock<T, F, Fut>(store: Arc<ECStore>, operation: F) -> S3Result<T>
where where
T: Send + 'static, T: Send + 'static,
F: FnOnce() -> Fut + Send + 'static, F: FnOnce() -> Fut + Send + 'static,
+1 -16
View File
@@ -64,9 +64,7 @@ mod ecstore_metrics {
} }
mod ecstore_notification { mod ecstore_notification {
pub(crate) use crate::storage::storage_api::ecstore_notification::{ pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys;
CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof,
};
} }
#[allow(unused_imports)] #[allow(unused_imports)]
@@ -114,10 +112,6 @@ pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds;
pub(crate) type TierType = ecstore_tier::tier_config::TierType; pub(crate) type TierType = ecstore_tier::tier_config::TierType;
pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError; pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError;
pub(crate) fn acquire_cross_pool_fence_fleet_proof() -> Option<ecstore_notification::CrossPoolFenceFleetProofToken> {
ecstore_notification::acquire_cross_pool_fence_fleet_proof()
}
pub(crate) mod runtime_sources { pub(crate) mod runtime_sources {
pub(crate) type DailyAllTierStats = super::DailyAllTierStats; pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
pub(crate) type ECStore = super::ECStore; pub(crate) type ECStore = super::ECStore;
@@ -302,15 +296,6 @@ pub(crate) mod metadata_sys {
super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
} }
pub(crate) async fn update_quota_if_incarnation(
bucket: &str,
data: Vec<u8>,
expected_incarnation_id: uuid::Uuid,
proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken,
) -> Result<OffsetDateTime> {
super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await
}
pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> { pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
} }
-32
View File
@@ -23,9 +23,6 @@
//! `ECStore` and one metadata-sys initialization exist per test binary. //! `ECStore` and one metadata-sys initialization exist per test binary.
use super::storage_api::test::bucket::metadata_sys; use super::storage_api::test::bucket::metadata_sys;
use super::storage_api::test::bucket::quota::BucketQuota;
use super::storage_api::test::bucket::quota::checker::QuotaChecker;
use super::storage_api::test::contract::bucket::MakeBucketOptions;
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions}; use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions};
use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth}; use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth};
@@ -90,17 +87,6 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone()) crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone())
.await .await
.expect("initialize notification system for gating test env"); .expect("initialize notification system for gating test env");
let topology_fingerprint =
crate::storage::storage_api::heal_control_startup_consumer::heal_topology_fingerprint(&endpoint_pools)
.expect("single-node gating topology should hash");
crate::storage::storage_api::start_remote_version_state_fleet_probe(topology_fingerprint);
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while crate::storage::storage_api::ecstore_notification::acquire_cross_pool_fence_fleet_proof().is_none() {
tokio::task::yield_now().await;
}
})
.await
.expect("single-node cross-pool fence capability proof should publish");
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -121,24 +107,6 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
ecstore ecstore
} }
pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc<ECStore>, String) {
let store = shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("{prefix:.30}-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create durable quota test bucket");
super::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 0).await;
let metadata_sys =
crate::app::storage_api::test::get_global_bucket_metadata_sys().expect("test app context should expose bucket metadata");
QuotaChecker::new(metadata_sys)
.set_quota_config(&bucket, BucketQuota::new(Some(limit)))
.await
.expect("configure durable quota test bucket");
(store, bucket)
}
pub(crate) async fn shared_gating_ambient() -> Arc<AppContext> { pub(crate) async fn shared_gating_ambient() -> Arc<AppContext> {
let store = shared_gating_ecstore().await; let store = shared_gating_ecstore().await;
if let Some(ambient) = crate::runtime_sources::current_app_context() { if let Some(ambient) = crate::runtime_sources::current_app_context() {
+84 -277
View File
@@ -27,20 +27,19 @@ use super::storage_api::multipart_usecase::bucket::{
replication::{must_replicate_object, schedule_object_replication}, replication::{must_replicate_object, schedule_object_replication},
versioning_sys::BucketVersioningSys, versioning_sys::BucketVersioningSys,
}; };
use super::storage_api::multipart_usecase::compression::{is_disk_compressible, is_multipart_disk_compression_enabled};
#[cfg(test)] #[cfg(test)]
use super::storage_api::multipart_usecase::contract::http::HTTPPreconditions; use super::storage_api::multipart_usecase::contract::http::HTTPPreconditions;
use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _, MultipartUploadResult}; use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _, MultipartUploadResult};
use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec; use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec;
use super::storage_api::multipart_usecase::data_usage::{ use super::storage_api::multipart_usecase::data_usage::{
quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
}; };
use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
use super::storage_api::multipart_usecase::helper::OperationHelper; use super::storage_api::multipart_usecase::helper::OperationHelper;
#[cfg(test)] #[cfg(test)]
use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value}; use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan};
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag; use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
use super::storage_api::multipart_usecase::options::{ use super::storage_api::multipart_usecase::options::{
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization, copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
@@ -65,10 +64,10 @@ use super::storage_api::multipart_usecase::{
}; };
use crate::app::object_data_cache::{ use crate::app::object_data_cache::{
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
invalidate_object_data_cache_before_mutation, invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
}; };
use crate::app::object_usecase::{ use crate::app::object_usecase::{
acquire_copy_bucket_lifecycle_locks, apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome,
validate_existing_object_lock_for_write, validate_existing_object_lock_for_write,
}; };
use crate::app::runtime_sources::{ use crate::app::runtime_sources::{
@@ -84,8 +83,6 @@ use rustfs_io_metrics::record_s3_op;
use rustfs_s3_ops::S3Operation; use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName; use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm; use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
use rustfs_utils::http::insert_header;
use rustfs_utils::http::{ use rustfs_utils::http::{
SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme,
@@ -211,28 +208,6 @@ fn create_multipart_upload_metadata(
metadata metadata
} }
/// A multipart session advertises disk compression only when the staged-rollout
/// switch (`RUSTFS_COMPRESSION_MULTIPART_ENABLED`) is on, the object key/headers
/// qualify, AND the session is not an SSE-C ciphertext-passthrough replication
/// session, which must preserve source bytes verbatim.
///
/// The rollout switch defaults to off so a rolling upgrade never creates new
/// compressed multipart objects while pre-fix nodes (whose decompressor is not
/// resumable) may still serve reads. Enable it once the fleet has converged on a
/// fixed build; the default flips per the `multipart-compression-default-off-window`
/// entry in docs/architecture/compat-cleanup-register.md.
///
/// Each part is compressed as an independent stream; the GET path decodes across part
/// boundaries (see `ReadTransform::Compressed`), so the session may advertise
/// object-level compression again.
///
/// Unlike single PUT there is no `MIN_DISK_COMPRESSIBLE_SIZE` floor here: the total
/// object size is unknown at CreateMultipartUpload time, so tiny multipart objects pay
/// the (harmless) framing overhead. This is a deliberate trade-off, not a bug.
fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_passthrough: bool, disk_compressible: bool) -> bool {
multipart_enabled && !ciphertext_passthrough && disk_compressible
}
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> { async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
table_catalog::validate_bucket_object_mutation(bucket, key) table_catalog::validate_bucket_object_mutation(bucket, key)
.await .await
@@ -251,8 +226,12 @@ fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
opts opts
} }
fn logical_object_size(info: &ObjectInfo) -> Result<u64, StorageError> {
u64::try_from(info.get_actual_size()?).map_err(|_| StorageError::PartMissingOrCorrupt)
}
fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> { fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> {
match quota_object_size(info) { match logical_object_size(info) {
Ok(size) => Ok(size), Ok(size) => Ok(size),
Err(err) if fail_closed => Err(ApiError::from(err).into()), Err(err) if fail_closed => Err(ApiError::from(err).into()),
Err(_) => Ok(info.size.max(0) as u64), Err(_) => Ok(info.size.max(0) as u64),
@@ -528,7 +507,11 @@ impl DefaultMultipartUsecase {
Ok(existing_obj_info) => { Ok(existing_obj_info) => {
validate_existing_object_lock_for_write(&existing_obj_info, &current_opts)?; validate_existing_object_lock_for_write(&existing_obj_info, &current_opts)?;
let physical_size = existing_obj_info.size.max(0) as u64; let physical_size = existing_obj_info.size.max(0) as u64;
let logical_size = quota_object_size(&existing_obj_info); let logical_size = if opts.replication_request {
Ok(physical_size)
} else {
logical_object_size(&existing_obj_info)
};
Some((physical_size, logical_size)) Some((physical_size, logical_size))
} }
Err(err) => { Err(err) => {
@@ -577,20 +560,32 @@ impl DefaultMultipartUsecase {
}; };
let quota_metadata_sys = self.bucket_metadata_sys(); let quota_metadata_sys = self.bucket_metadata_sys();
let quota_tracking = quota_metadata_sys.is_some();
let mut quota_enabled = false;
if let Some(metadata_sys) = quota_metadata_sys.as_ref() { if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
let quota_checker = QuotaChecker::new(metadata_sys.clone()); let quota_checker = QuotaChecker::new(metadata_sys.clone());
let check_result = let check_result =
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
quota_enabled = check_result.quota_limit.is_some(); // Ciphertext-passthrough replication parts use a different size basis and retain
apply_quota_admission(&mut opts, &check_result)?; // the existing post-commit accounting path until they carry a trusted logical-size proof.
if !opts.replication_request
&& let Some(quota_limit) = check_result.quota_limit
{
let installed = check_result
.current_usage
.is_some_and(|current_usage| opts.set_quota_admission(current_usage, quota_limit));
if !installed {
return Err(S3Error::with_message(
S3ErrorCode::ServiceUnavailable,
"Bucket quota check temporarily unavailable, please retry".to_string(),
));
}
}
} }
let previous_current_size = match previous_current_sizes { let previous_current_size = match previous_current_sizes {
Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), Some((physical_size, _)) if opts.replication_request => Some(physical_size),
Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), Some((_, Ok(logical_size))) => Some(logical_size),
Some((physical_size, _)) => Some(physical_size), Some((_, Err(err))) if opts.quota_admission.is_some() => return Err(ApiError::from(err).into()),
Some((physical_size, Err(_))) => Some(physical_size),
None => None, None => None,
}; };
@@ -600,6 +595,7 @@ impl DefaultMultipartUsecase {
let key = key.clone(); let key = key.clone();
let upload_id = upload_id.clone(); let upload_id = upload_id.clone();
let opts = opts.clone(); let opts = opts.clone();
let quota_metadata_sys = quota_metadata_sys.clone();
async move { async move {
let obj_info = store let obj_info = store
.clone() .clone()
@@ -609,9 +605,37 @@ impl DefaultMultipartUsecase {
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
record_capacity_write(Some(capacity_scope_token)).await; record_capacity_write(Some(capacity_scope_token)).await;
if quota_tracking { if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; if opts.replication_request {
let quota_checker = QuotaChecker::new(metadata_sys.clone());
match quota_checker
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64)
.await
{
Ok(check_result) if !check_result.allowed => {
let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes",
check_result.current_usage.unwrap_or(0),
check_result.quota_limit.unwrap_or(0)
),
));
}
Err(err) => {
warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err);
}
Ok(_) => {}
}
}
let committed_size = if opts.replication_request {
obj_info.size.max(0) as u64
} else {
quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())?
};
if versioned { if versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
} else { } else {
@@ -821,20 +845,6 @@ impl DefaultMultipartUsecase {
let ciphertext_passthrough = replication_authorized let ciphertext_passthrough = replication_authorized
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") && get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some(); && rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
let check_result = map_quota_check_outcome(
&bucket,
QuotaChecker::new(metadata_sys)
.check_quota(&bucket, QuotaOperation::PutObject, 0)
.await,
)?;
if check_result.quota_limit.is_some() {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
"SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(),
));
}
}
if ciphertext_passthrough { if ciphertext_passthrough {
insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string()); insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string());
} }
@@ -860,17 +870,8 @@ impl DefaultMultipartUsecase {
None => (None, None), None => (None, None),
}; };
if should_advertise_session_compression( // Multipart parts are independent physical streams. Advertising object-level
is_multipart_disk_compression_enabled(), // compression here would make GET decode the completed object as one stream.
ciphertext_passthrough,
is_disk_compressible(&req.headers, &key),
) {
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
compression_metadata_value(CompressionAlgorithm::default()),
);
}
let mt2 = metadata.clone(); let mt2 = metadata.clone();
let mut opts: ObjectOptions = let mut opts: ObjectOptions =
@@ -1664,31 +1665,6 @@ mod tests {
DefaultMultipartUsecase::without_context() DefaultMultipartUsecase::without_context()
} }
#[test]
fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() {
// (multipart_enabled, ciphertext_passthrough, disk_compressible, expected)
let cases = [
(true, false, false, false),
(true, false, true, true),
(true, true, false, false),
(true, true, true, false),
// The staged-rollout switch keeps multipart compression dark by
// default regardless of the other gates.
(false, false, true, false),
(false, false, false, false),
(false, true, true, false),
(false, true, false, false),
];
for (multipart_enabled, ciphertext_passthrough, disk_compressible, expected) in cases {
assert_eq!(
should_advertise_session_compression(multipart_enabled, ciphertext_passthrough, disk_compressible),
expected,
"multipart_enabled={multipart_enabled} ciphertext_passthrough={ciphertext_passthrough} disk_compressible={disk_compressible}"
);
}
}
#[test] #[test]
fn quota_accounting_uses_logical_size_when_available() { fn quota_accounting_uses_logical_size_when_available() {
let mut metadata = HashMap::new(); let mut metadata = HashMap::new();
@@ -1702,24 +1678,6 @@ mod tests {
assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192); assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192);
assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192); assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192);
let mut poisoned_metadata = HashMap::new();
insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
let poisoned = ObjectInfo {
size: 17,
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 4096,
actual_size: 4096,
..Default::default()
}]),
user_defined: Arc::new(poisoned_metadata),
..Default::default()
};
assert_eq!(
quota_accounting_object_size(&poisoned, true).expect("persisted part size must be charged"),
4096
);
} }
#[test] #[test]
@@ -2128,14 +2086,30 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial_test::serial] #[serial_test::serial]
async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() { async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("compressed-complete-quota", 16_384).await; use crate::app::storage_api::multipart_usecase::bucket::quota::BucketQuota;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("compressed-complete-quota-{}", Uuid::new_v4());
let object = "object"; let object = "object";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create compressed quota bucket");
seed_bucket_usage_memory_for_test(&bucket, 0).await;
let usecase = DefaultMultipartUsecase::from_global(); let usecase = DefaultMultipartUsecase::from_global();
let metadata_sys = usecase let metadata_sys = usecase
.bucket_metadata_sys() .bucket_metadata_sys()
.expect("test app context should expose bucket metadata"); .expect("test app context should expose bucket metadata");
let quota_checker = QuotaChecker::new(metadata_sys); let mut quota_checker = QuotaChecker::new(metadata_sys);
quota_checker
.set_quota_config(&bucket, BucketQuota::new(Some(16_384)))
.await
.expect("configure bucket quota");
for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] { for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] {
let mut create_opts = ObjectOptions::default(); let mut create_opts = ObjectOptions::default();
@@ -2180,173 +2154,6 @@ mod tests {
} }
} }
#[tokio::test]
#[serial_test::serial]
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
let usecase = DefaultMultipartUsecase::from_global();
let input = CreateMultipartUploadInput::builder()
.bucket(bucket)
.key("object".to_string())
.build()
.expect("create multipart request should build");
let mut request = build_request(input, Method::POST);
insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
request
.headers
.insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256"));
request.extensions.insert(crate::storage::access::ReqInfo {
replication_request_authorized: true,
..Default::default()
});
let err = usecase
.execute_create_multipart_upload(request)
.await
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
#[tokio::test]
#[serial_test::serial]
async fn concurrent_completions_share_durable_bucket_quota_reservations() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-complete-quota", 6000).await;
let usecase = DefaultMultipartUsecase::from_global();
let mut inputs = Vec::new();
for object in ["first", "second"] {
let upload = store
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
.await
.expect("create concurrent multipart upload");
let mut reader = PutObjReader::from_vec(vec![0x71; 4096]);
let part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
.await
.expect("stage concurrent multipart part");
inputs.push(
CompleteMultipartUploadInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.upload_id(upload.upload_id)
.multipart_upload(Some(CompletedMultipartUpload {
parts: Some(vec![CompletedPart {
part_number: Some(1),
e_tag: part.etag.map(|etag| to_s3s_etag(&etag)),
..Default::default()
}]),
}))
.build()
.expect("build concurrent completion input"),
);
}
let first_usecase = usecase.clone();
let first = first_usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST));
let second = usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST));
let (first, second) = tokio::join!(first, second);
assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1);
let denied = first.err().or_else(|| second.err()).expect("one completion must be denied");
assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest);
}
#[tokio::test]
#[serial_test::serial]
async fn multipart_completion_rejects_rotated_quota_capability_before_rename() {
use crate::app::storage_api::test::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-mpu-quota", 4096).await;
let object = "object";
let upload = store
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
.await
.expect("create multipart upload");
let mut reader = PutObjReader::from_vec(vec![0x78; 4096]);
let part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
.await
.expect("stage multipart part");
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename);
let complete_store = Arc::clone(&store);
let complete_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(
&complete_bucket,
object,
&upload_id,
vec![CompletePart {
part_num: 1,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
});
barrier.wait_until_paused().await;
assert!(
crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(),
"the gating environment must have a current fleet proof"
);
barrier.release();
let err = complete
.await
.expect("completion task should not panic")
.expect_err("a replaced fleet proof must fence multipart rename");
assert!(matches!(
err,
StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
..
}
));
store
.get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
.await
.expect("proof rotation must preserve the multipart upload for retry");
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_multipart_completion_has_zero_quota_growth() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-mpu-quota", 0).await;
let object = "object";
let mut movement_opts = ObjectOptions {
data_movement: true,
..Default::default()
};
let upload = store
.new_multipart_upload(&bucket, object, &movement_opts)
.await
.expect("create data-movement multipart upload");
let mut reader = PutObjReader::from_vec(vec![0x7a; 4096]);
let part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &movement_opts)
.await
.expect("stage data-movement multipart part");
movement_opts.preserve_etag = Some("movement-etag".to_string());
let completed = store
.complete_multipart_upload(
&bucket,
object,
&upload.upload_id,
vec![CompletePart {
part_num: 1,
etag: part.etag,
..Default::default()
}],
&movement_opts,
)
.await
.expect("moving an already-accounted multipart object between pools must have zero quota growth");
assert_eq!(completed.size, 4096);
}
#[tokio::test] #[tokio::test]
#[serial_test::serial] #[serial_test::serial]
async fn rejected_empty_parts_preserve_existing_object_and_staging() { async fn rejected_empty_parts_preserve_existing_object_and_staging() {
File diff suppressed because it is too large Load Diff
+2 -16
View File
@@ -47,8 +47,6 @@ pub(crate) mod capacity {
pub(crate) mod data_usage { pub(crate) mod data_usage {
use std::sync::Arc; use std::sync::Arc;
pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size;
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
} }
@@ -942,9 +940,7 @@ pub(crate) mod concurrency {
} }
pub(crate) mod compression { pub(crate) mod compression {
pub(crate) use crate::storage::storage_api::ecstore_compression::{ pub(crate) use crate::storage::storage_api::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_multipart_disk_compression_enabled,
};
} }
pub(crate) mod deadlock_detector { pub(crate) mod deadlock_detector {
@@ -1155,7 +1151,7 @@ pub(crate) mod multipart_usecase {
} }
pub(crate) use super::{ pub(crate) use super::{
access, bucket, compression, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, access, bucket, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse,
}; };
pub(crate) use crate::storage::storage_api::{ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader}; pub(crate) use crate::storage::storage_api::{ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader};
} }
@@ -1218,14 +1214,4 @@ pub(crate) mod test {
pub(crate) use crate::storage::storage_api::{ pub(crate) use crate::storage::storage_api::{
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
}; };
pub(crate) mod set_disk {
pub(crate) use crate::storage::storage_api::ecstore_set_disk::{
MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause,
fail_next_quota_ledger_save_for_test,
};
}
pub(crate) mod metadata_sys {
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::ConfigWriteLockProbe;
}
} }
+4 -32
View File
@@ -326,11 +326,7 @@ impl From<StorageError> for ApiError {
_ => S3ErrorCode::InternalError, _ => S3ErrorCode::InternalError,
}; };
let message = if matches!(&err, StorageError::QuotaExceeded { .. }) { let message = if matches!(&err, StorageError::QuotaExceeded { .. }) || code == S3ErrorCode::InternalError {
err.to_string()
} else if code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)) {
ApiError::error_code_to_message(&code)
} else if code == S3ErrorCode::InternalError {
err.to_string() err.to_string()
} else if let StorageError::InvalidArgument(_, _, reason) = &err } else if let StorageError::InvalidArgument(_, _, reason) = &err
&& !reason.is_empty() && !reason.is_empty()
@@ -529,25 +525,6 @@ mod tests {
assert!(api_error.source.is_some()); assert!(api_error.source.is_some());
} }
#[test]
fn storage_io_internal_error_redacts_public_message_and_retains_source() {
let sensitive_path = "/sensitive/storage/path";
let api_error = ApiError::from(StorageError::Io(IoError::new(
ErrorKind::PermissionDenied,
format!("permission denied: {sensitive_path}"),
)));
assert_eq!(api_error.code, S3ErrorCode::InternalError);
assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
assert!(!api_error.message.contains(sensitive_path));
let source = api_error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains(sensitive_path)));
}
#[test] #[test]
fn test_kms_service_unavailable_maps_to_retryable_error() { fn test_kms_service_unavailable_maps_to_retryable_error() {
let api_error = ApiError::from(StorageError::other(KmsUnavailableError)); let api_error = ApiError::from(StorageError::other(KmsUnavailableError));
@@ -714,14 +691,9 @@ mod tests {
let iam_error = rustfs_iam::error::Error::other("IAM test error"); let iam_error = rustfs_iam::error::Error::other("IAM test error");
let api_error: ApiError = iam_error.into(); let api_error: ApiError = iam_error.into();
assert_eq!(api_error.code, S3ErrorCode::InternalError); // IAM error is first converted to StorageError, then to ApiError
assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); assert!(api_error.source.is_some());
let source = api_error assert!(api_error.message.contains("test error"));
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("IAM test error")));
} }
#[test] #[test]
+3 -3
View File
@@ -301,7 +301,7 @@ impl S3 for FS {
#[instrument(level = "debug", skip(self, req))] #[instrument(level = "debug", skip(self, req))]
async fn copy_object(&self, req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> { async fn copy_object(&self, req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> {
let usecase = s3_api::object_usecase_for(self); let usecase = s3_api::object_usecase_for(self);
usecase.execute_copy_object(req).await Box::pin(usecase.execute_copy_object(req)).await
} }
#[instrument( #[instrument(
@@ -704,7 +704,7 @@ impl S3 for FS {
async fn get_object(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> { async fn get_object(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
crate::hp_guard!("S3::get_object"); crate::hp_guard!("S3::get_object");
let usecase = s3_api::object_usecase_for(self); let usecase = s3_api::object_usecase_for(self);
usecase.execute_get_object(req).await Box::pin(usecase.execute_get_object(req)).await
} }
async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> { async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> {
@@ -1261,7 +1261,7 @@ impl S3 for FS {
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> { async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
crate::hp_guard!("S3::put_object"); crate::hp_guard!("S3::put_object");
let usecase = s3_api::object_usecase_for(self); let usecase = s3_api::object_usecase_for(self);
usecase.execute_put_object(self, req).await Box::pin(usecase.execute_put_object(self, req)).await
} }
async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> { async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
+9 -7
View File
@@ -153,7 +153,9 @@ fn remove_heal_control_replay(
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new(); static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; // RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition,
// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0;
fn admit_heal_control_replay( fn admit_heal_control_replay(
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>, replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -2943,7 +2945,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn snapshot_lease_acquire_and_renew_handlers_fail_closed_for_missing_disk() { async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() {
let service = make_server(); let service = make_server();
let disk = "http://node-a:9000/data/rustfs0".to_string(); let disk = "http://node-a:9000/data/rustfs0".to_string();
@@ -2960,7 +2962,7 @@ mod tests {
let acquire = service let acquire = service
.acquire_snapshot_lease(acquire) .acquire_snapshot_lease(acquire)
.await .await
.expect("missing-disk acquire should return a protocol response") .expect("disabled acquire should return a protocol response")
.into_inner(); .into_inner();
let mut renew = Request::new(SnapshotLeaseRenewRequest { let mut renew = Request::new(SnapshotLeaseRenewRequest {
@@ -2977,14 +2979,14 @@ mod tests {
let renew = service let renew = service
.renew_snapshot_lease(renew) .renew_snapshot_lease(renew)
.await .await
.expect("missing-disk renew should return a protocol response") .expect("disabled renew should return a protocol response")
.into_inner(); .into_inner();
for response in [acquire, renew] { for response in [acquire, renew] {
assert!(!response.success); assert!(!response.success);
assert!(response.token.is_empty()); assert!(response.token.is_empty());
assert_eq!(response.protocol_version, 1); assert_eq!(response.protocol_version, 1);
assert_eq!(response.error, Some(DiskError::other("cannot find disk").into())); assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into()));
} }
} }
@@ -3340,7 +3342,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn cross_pool_fence_probe_authenticates_supported_v1_state() { async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() {
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
assert!( assert!(
@@ -3405,7 +3407,7 @@ mod tests {
assert!(response.success); assert!(response.success);
assert_eq!(response.error_info, None); assert_eq!(response.error_info, None);
assert_eq!(&response.result[..4], &1_u32.to_be_bytes()); assert_eq!(&response.result[..4], &0_u32.to_be_bytes());
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..]) let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
.expect("capability identity should decode"); .expect("capability identity should decode");
assert_eq!(topology_member, "node-a:9000"); assert_eq!(topology_member, "node-a:9000");
+26 -39
View File
@@ -37,28 +37,19 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
fn snapshot_lease_response(result: Result<SnapshotLeaseToken, DiskError>) -> Response<SnapshotLeaseResponse> {
match result {
Ok(token) => Response::new(SnapshotLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: None,
}),
Err(err) => Response::new(SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(err.into()),
}),
}
}
struct DecodedRpcPayload<T> { struct DecodedRpcPayload<T> {
value: T, value: T,
from_msgpack: bool, from_msgpack: bool,
} }
fn snapshot_lease_disabled_response() -> SnapshotLeaseResponse {
SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(DiskError::UnsupportedDisk.into()),
}
}
fn decode_msgpack_or_json<T: DeserializeOwned>( fn decode_msgpack_or_json<T: DeserializeOwned>(
binary: &[u8], binary: &[u8],
json: &str, json: &str,
@@ -250,12 +241,7 @@ impl NodeService {
rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()), rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()),
"acquire_snapshot_lease", "acquire_snapshot_lease",
)?; )?;
let request = request.into_inner(); Ok(Response::new(snapshot_lease_disabled_response()))
let result = match self.find_disk(&request.disk).await {
Some(disk) => disk.acquire_snapshot_lease(&request.volume, &request.path).await,
None => Err(DiskError::other("cannot find disk")),
};
Ok(snapshot_lease_response(result))
} }
pub(super) async fn handle_renew_snapshot_lease( pub(super) async fn handle_renew_snapshot_lease(
@@ -267,14 +253,7 @@ impl NodeService {
rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()), rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()),
"renew_snapshot_lease", "renew_snapshot_lease",
)?; )?;
let request = request.into_inner(); Ok(Response::new(snapshot_lease_disabled_response()))
let token =
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
let result = match self.find_disk(&request.disk).await {
Some(disk) => disk.renew_snapshot_lease(&request.volume, &request.path, token).await,
None => Err(DiskError::other("cannot find disk")),
};
Ok(snapshot_lease_response(result))
} }
pub(super) async fn handle_release_snapshot_lease( pub(super) async fn handle_release_snapshot_lease(
@@ -287,11 +266,8 @@ impl NodeService {
"release_snapshot_lease", "release_snapshot_lease",
)?; )?;
let request = request.into_inner(); let request = request.into_inner();
let token = if request.token.as_ref() == SnapshotLeaseToken::revoke_all().as_bytes() { let token =
SnapshotLeaseToken::revoke_all() SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
} else {
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?
};
let Some(disk) = self.find_disk(&request.disk).await else { let Some(disk) = self.find_disk(&request.disk).await else {
return Ok(Response::new(SnapshotLeaseMutationResponse { return Ok(Response::new(SnapshotLeaseMutationResponse {
success: false, success: false,
@@ -1049,7 +1025,7 @@ impl NodeService {
.rename_data( .rename_data(
&request.src_volume, &request.src_volume,
&request.src_path, &request.src_path,
&decoded_file_info.value, decoded_file_info.value,
&request.dst_volume, &request.dst_volume,
&request.dst_path, &request.dst_path,
) )
@@ -1518,11 +1494,11 @@ mod tests {
use super::{ use super::{
compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info,
encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named,
encode_read_multiple_response_payloads, encode_rename_data_response_payloads, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, snapshot_lease_disabled_response,
}; };
use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::ReadMultipleResp;
use crate::storage::storage_api::RenameDataResp;
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
use crate::storage::storage_api::{DiskError, RenameDataResp};
use rustfs_filemeta::FileInfo; use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -1533,6 +1509,17 @@ mod tests {
count: u32, count: u32,
} }
#[test]
fn snapshot_lease_acquire_and_renew_fail_closed() {
let response = snapshot_lease_disabled_response();
let expected_error = DiskError::UnsupportedDisk.into();
assert!(!response.success);
assert!(response.token.is_empty());
assert_eq!(response.protocol_version, 1);
assert_eq!(response.error, Some(expected_error));
}
#[test] #[test]
fn decode_msgpack_or_json_prefers_binary_payload() { fn decode_msgpack_or_json_prefers_binary_payload() {
let payload = SamplePayload { let payload = SamplePayload {
+1 -123
View File
@@ -39,11 +39,6 @@ pub(crate) struct ListMultipartUploadsParams {
pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput { pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput {
let owner = rustfs_owner(); let owner = rustfs_owner();
let initiator = rustfs_initiator(); let initiator = rustfs_initiator();
let transformed_parts = rustfs_utils::http::contains_key_str(&res.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION)
|| res
.user_defined
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key));
ListPartsOutput { ListPartsOutput {
bucket: Some(res.bucket), bucket: Some(res.bucket),
@@ -56,14 +51,7 @@ pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput {
e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), e_tag: p.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: p.last_mod.map(Timestamp::from), last_modified: p.last_mod.map(Timestamp::from),
part_number: p.part_num.try_into().ok(), part_number: p.part_num.try_into().ok(),
// Compressed parts store fewer bytes than the client sent; S3 size: p.size.try_into().ok(),
// semantics report the uploaded (logical) size, matching
// GetObjectAttributes ObjectParts.
size: if p.actual_size > 0 || (transformed_parts && p.actual_size == 0) {
Some(p.actual_size)
} else {
p.size.try_into().ok()
},
..Default::default() ..Default::default()
}) })
.collect(), .collect(),
@@ -259,116 +247,6 @@ mod tests {
assert_eq!(output.initiator, Some(rustfs_initiator())); assert_eq!(output.initiator, Some(rustfs_initiator()));
} }
#[test]
fn test_list_parts_output_reports_logical_size_for_compressed_parts() {
let input = ListPartsInfo {
bucket: "bucket-a".to_string(),
object: "obj-a".to_string(),
upload_id: "upload-a".to_string(),
parts: vec![PartInfo {
part_num: 1,
// Stored (compressed) bytes on disk vs. the logical size the client uploaded.
size: 1_024,
actual_size: 8_388_608,
..Default::default()
}],
..Default::default()
};
let output = build_list_parts_output(input);
let parts = output.parts.as_ref().expect("parts should be present");
assert_eq!(parts.len(), 1);
assert_eq!(
parts[0].size,
Some(8_388_608),
"compressed parts must report the uploaded logical size, not the stored size"
);
}
#[test]
fn test_list_parts_output_reports_zero_logical_size_for_compressed_parts() {
let mut user_defined = std::collections::HashMap::new();
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
let input = ListPartsInfo {
user_defined,
parts: vec![PartInfo {
part_num: 1,
// Legacy SSE writes an 8-byte end record for an empty part.
size: 8,
actual_size: 0,
..Default::default()
}],
..Default::default()
};
let output = build_list_parts_output(input);
let parts = output.parts.as_ref().expect("parts should be present");
assert_eq!(parts[0].size, Some(0));
}
#[test]
fn test_list_parts_output_reports_zero_logical_size_for_encrypted_parts() {
let input = ListPartsInfo {
user_defined: std::collections::HashMap::from([(
rustfs_utils::http::AMZ_SERVER_SIDE_ENCRYPTION.to_string(),
"AES256".to_string(),
)]),
parts: vec![
PartInfo {
part_num: 1,
size: 8,
actual_size: 0,
..Default::default()
},
PartInfo {
part_num: 2,
size: 8,
actual_size: -1,
..Default::default()
},
],
..Default::default()
};
let output = build_list_parts_output(input);
let parts = output.parts.as_ref().expect("parts should be present");
assert_eq!(parts[0].size, Some(0));
assert_eq!(parts[1].size, Some(8));
}
#[test]
fn test_list_parts_output_falls_back_to_stored_size_when_actual_size_unknown() {
let input = ListPartsInfo {
parts: vec![
PartInfo {
part_num: 1,
size: 1_024,
// Uncompressed parts leave actual_size unset.
actual_size: 0,
..Default::default()
},
PartInfo {
part_num: 2,
size: 1_024,
// Legacy/unknown sentinel must not leak a negative size to clients.
actual_size: -1,
..Default::default()
},
],
..Default::default()
};
let output = build_list_parts_output(input);
let parts = output.parts.as_ref().expect("parts should be present");
assert_eq!(parts.len(), 2);
assert_eq!(parts[0].size, Some(1024));
assert_eq!(parts[1].size, Some(1024));
}
#[test] #[test]
fn test_list_parts_output_normalizes_legacy_storage_class_and_handles_overflow_markers() { fn test_list_parts_output_normalizes_legacy_storage_class_and_handles_overflow_markers() {
let input = ListPartsInfo { let input = ListPartsInfo {
+8 -21
View File
@@ -4565,9 +4565,11 @@ mod tests {
}) })
.await .await
.expect_err("mismatched kms context should fail"); .expect_err("mismatched kms context should fail");
assert_eq!(err.code, S3ErrorCode::InternalError); assert!(
assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); err.message.contains("context") || err.message.contains("Context"),
assert_eq!(super::kms_data_plane_error_class(&err), "context_mismatch"); "unexpected error for mismatched kms context: {}",
err.message
);
manager.stop().await.expect("kms service should stop cleanly"); manager.stop().await.expect("kms service should stop cleanly");
reset_sse_dek_provider(); reset_sse_dek_provider();
@@ -5334,16 +5336,7 @@ mod tests {
let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32]) let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32])
.expect_err("unknown JSON envelope versions must fail closed"); .expect_err("unknown JSON envelope versions must fail closed");
assert_eq!(error.code, S3ErrorCode::InternalError); assert!(error.message.contains("Unsupported encrypted DEK format version"));
assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
let source = error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error
.to_string()
.contains("Unsupported encrypted DEK format version")));
} }
#[tokio::test] #[tokio::test]
@@ -5901,16 +5894,10 @@ mod tests {
} }
#[test] #[test]
fn test_map_get_object_reader_error_redacts_non_ssec_internal_errors() { fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
let err = map_get_object_reader_error(StorageError::other("plain io failure")); let err = map_get_object_reader_error(StorageError::other("plain io failure"));
assert_eq!(err.code, S3ErrorCode::InternalError); assert_eq!(err.code, S3ErrorCode::InternalError);
assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); assert_eq!(err.message, "Io error: plain io failure");
let source = err
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("plain io failure")));
} }
#[test] #[test]
+6 -27
View File
@@ -409,9 +409,7 @@ pub(crate) mod ecstore_client {
} }
pub(crate) mod ecstore_compression { pub(crate) mod ecstore_compression {
pub(crate) use rustfs_ecstore::api::compression::{ pub(crate) use rustfs_ecstore::api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_multipart_disk_compression_enabled,
};
} }
pub(crate) mod ecstore_cluster { pub(crate) mod ecstore_cluster {
@@ -432,7 +430,7 @@ pub(crate) mod ecstore_config {
pub(crate) mod ecstore_data_usage { pub(crate) mod ecstore_data_usage {
pub(crate) use rustfs_ecstore::api::data_usage::{ pub(crate) use rustfs_ecstore::api::data_usage::{
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached, apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
}; };
@@ -489,12 +487,8 @@ pub(crate) mod ecstore_metrics {
#[allow(unused_imports)] #[allow(unused_imports)]
pub(crate) mod ecstore_notification { pub(crate) mod ecstore_notification {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::rotate_cross_pool_fence_fleet_proof_for_test;
pub(crate) use rustfs_ecstore::api::notification::{ pub(crate) use rustfs_ecstore::api::notification::{
CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
}; };
} }
@@ -552,11 +546,6 @@ pub(crate) mod ecstore_test_support {
} }
pub(crate) mod ecstore_set_disk { pub(crate) mod ecstore_set_disk {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause,
fail_next_quota_ledger_save_for_test,
};
pub(crate) use rustfs_ecstore::api::set_disk::{ pub(crate) use rustfs_ecstore::api::set_disk::{
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
}; };
@@ -1141,9 +1130,7 @@ pub(crate) trait StorageDiskRpcExt {
) -> DiskResult<()>; ) -> DiskResult<()>;
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>; async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>;
async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>;
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken>;
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>; async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>;
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken>;
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>; async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>;
async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>; async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>;
async fn make_volume(&self, volume: &str) -> DiskResult<()>; async fn make_volume(&self, volume: &str) -> DiskResult<()>;
@@ -1152,7 +1139,7 @@ pub(crate) trait StorageDiskRpcExt {
&self, &self,
src_volume: &str, src_volume: &str,
src_path: &str, src_path: &str,
file_info: &rustfs_filemeta::FileInfo, file_info: rustfs_filemeta::FileInfo,
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> DiskResult<RenameDataResp>; ) -> DiskResult<RenameDataResp>;
@@ -1271,18 +1258,10 @@ where
ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await
} }
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken> {
ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await
}
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> { async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> {
ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await
} }
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken> {
ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await
}
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> { async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> {
ecstore_disk::DiskAPI::stat_volume(self, volume).await ecstore_disk::DiskAPI::stat_volume(self, volume).await
} }
@@ -1303,11 +1282,11 @@ where
&self, &self,
src_volume: &str, src_volume: &str,
src_path: &str, src_path: &str,
file_info: &rustfs_filemeta::FileInfo, file_info: rustfs_filemeta::FileInfo,
dst_volume: &str, dst_volume: &str,
dst_path: &str, dst_path: &str,
) -> DiskResult<RenameDataResp> { ) -> DiskResult<RenameDataResp> {
ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info.clone(), dst_volume, dst_path).await ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await
} }
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult<Vec<String>> { async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult<Vec<String>> {
+4 -635
View File
@@ -26,7 +26,10 @@ use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc; use std::sync::Arc;
use time::OffsetDateTime; use time::OffsetDateTime;
use super::*; use super::{
StrongTableCatalogRuntime, TableCatalogObject, TableCatalogObjectBackend, TableCatalogObjectMetadata,
TableCatalogPutPrecondition, TableCatalogStoreError, TableCatalogStoreResult, TableCommitPublication,
};
pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_json::Value { pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_json::Value {
serde_json::json!({ serde_json::json!({
@@ -936,637 +939,3 @@ impl TestCatalogObjectBackend {
.expect("lock acquisition attempts should be observable"); .expect("lock acquisition attempts should be observable");
} }
} }
#[derive(Clone, Default)]
pub(crate) struct TestCatalogPublishPause {
started: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
}
impl TestCatalogPublishPause {
pub(crate) async fn wait_started(&self) {
self.started.notified().await;
}
pub(crate) fn release(&self) {
self.release.notify_one();
}
}
// --- TableCatalogStore test doubles (backlog#1837 PR3) ---
//
// Two deliberately different shapes, per the issue's ruling: NoopTableCatalogStore
// is a pure stub whose methods answer "nothing here", used where a store must
// exist but never matter; TestTableCatalogStore is a stateful fake with commit
// pauses and failure injection. Both live here so a TableCatalogStore trait
// change is one file to update instead of two.
pub(crate) struct NoopTableCatalogStore;
#[async_trait::async_trait]
impl TableCatalogStore for NoopTableCatalogStore {
async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<Option<TableBucketEntry>> {
Ok(None)
}
async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult<Vec<NamespaceEntry>> {
Ok(Vec::new())
}
async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Option<NamespaceEntry>> {
Ok(None)
}
async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn register_table_with_publication(
&self,
entry: TableEntry,
publication: &(dyn TableCommitPublication + Sync),
) -> TableCatalogStoreResult<()> {
publication.begin_table_bucket(&entry.table_bucket).await?;
if !publication.holds_table_bucket(&entry.table_bucket) {
return Err(TableCatalogStoreError::Internal(
"table registration requires a table-bucket publication fence".to_string(),
));
}
let _publication_completion = TableCommitPublicationCompletion::new(publication);
publication
.prepare(&entry.table_bucket, &entry.namespace, &entry.table)
.await?;
if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) {
return Err(TableCatalogStoreError::Internal(
"table registration requires a table publication fence".to_string(),
));
}
self.register_table(entry).await
}
async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
Ok(Vec::new())
}
async fn list_all_tables(&self, _table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
Ok(Vec::new())
}
async fn load_table(
&self,
_table_bucket: &str,
_namespace: &str,
_table: &str,
) -> TableCatalogStoreResult<Option<TableEntry>> {
Ok(None)
}
async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult<TableCommitResult> {
let table = TableEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: request.table_bucket,
namespace: request.namespace,
table: request.table,
table_id: "table-id".to_string(),
table_uuid: "table-uuid".to_string(),
format: "ICEBERG".to_string(),
format_version: 2,
warehouse_location: "s3://analytics/tables/table-id".to_string(),
metadata_location: request.new_metadata_location.clone(),
version_token: "token-v2".to_string(),
generation: 2,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
};
let commit_log = CommitLogEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
commit_id: request.commit_id,
idempotency_key: request.idempotency_key,
table_id: table.table_id.clone(),
operation: request.operation,
expected_version_token: request.expected_version_token,
new_version_token: table.version_token.clone(),
previous_metadata_location: request.expected_metadata_location,
new_metadata_location: table.metadata_location.clone(),
requirements: request.requirements,
status: CommitLogStatus::Committed,
writer: request.writer,
created_at: None,
updated_at: None,
};
Ok(TableCommitResult { table, commit_log })
}
async fn commit_table_with_publication(
&self,
request: TableCommitRequest,
publication: &(dyn TableCommitPublication + Sync),
) -> TableCatalogStoreResult<TableCommitResult> {
publication
.prepare(&request.table_bucket, &request.namespace, &request.table)
.await?;
if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) {
return Err(TableCatalogStoreError::Internal(
"table commit requires a table publication fence".to_string(),
));
}
let _publication_completion = TableCommitPublicationCompletion::new(publication);
self.commit_table(request).await
}
async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Vec<ViewEntry>> {
Ok(Vec::new())
}
async fn load_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<Option<ViewEntry>> {
Ok(None)
}
async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult<ViewCommitResult> {
Ok(ViewCommitResult {
view: ViewEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: request.table_bucket,
namespace: request.namespace,
view: request.view,
view_id: "view-id".to_string(),
view_uuid: "view-uuid".to_string(),
format: "ICEBERG_VIEW".to_string(),
format_version: 1,
warehouse_location: "s3://analytics/views/view-id".to_string(),
metadata_location: request.new_metadata_location,
version_token: "token-v2".to_string(),
generation: 2,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
},
})
}
async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn get_commit_by_id(
&self,
_table_bucket: &str,
_table_id: &str,
_commit_id: &str,
) -> TableCatalogStoreResult<Option<CommitLogEntry>> {
Ok(None)
}
async fn get_commit_by_idempotency_key(
&self,
_table_bucket: &str,
_table_id: &str,
_idempotency_key: &str,
) -> TableCatalogStoreResult<Option<CommitLogEntry>> {
Ok(None)
}
}
#[derive(Default)]
pub(crate) struct TestTableCatalogStore {
pub(crate) table_buckets: tokio::sync::Mutex<Vec<crate::table_catalog::TableBucketEntry>>,
pub(crate) namespaces: tokio::sync::Mutex<Vec<crate::table_catalog::NamespaceEntry>>,
pub(crate) tables: tokio::sync::Mutex<Vec<crate::table_catalog::TableEntry>>,
pub(crate) views: tokio::sync::Mutex<Vec<crate::table_catalog::ViewEntry>>,
pub(crate) commits: tokio::sync::Mutex<Vec<crate::table_catalog::CommitLogEntry>>,
pub(crate) fail_put_table_bucket: tokio::sync::Mutex<bool>,
pub(crate) register_table_pause: Option<TestCatalogPublishPause>,
pub(crate) commit_table_pause: Option<TestCatalogPublishPause>,
}
#[async_trait::async_trait]
impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore {
async fn get_table_bucket(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::TableBucketEntry>> {
Ok(self
.table_buckets
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket)
.cloned())
}
async fn put_table_bucket(
&self,
entry: crate::table_catalog::TableBucketEntry,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await;
if *fail_put_table_bucket {
*fail_put_table_bucket = false;
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"injected table bucket write failure".to_string(),
));
}
drop(fail_put_table_bucket);
let mut table_buckets = self.table_buckets.lock().await;
table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket);
table_buckets.push(entry);
Ok(())
}
async fn create_namespace(
&self,
entry: crate::table_catalog::NamespaceEntry,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
self.namespaces.lock().await.push(entry);
Ok(())
}
async fn list_namespaces(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::NamespaceEntry>> {
Ok(self
.namespaces
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket)
.cloned()
.collect())
}
async fn get_namespace(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::NamespaceEntry>> {
Ok(self
.namespaces
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned())
}
async fn update_namespace_properties(
&self,
table_bucket: &str,
namespace: &str,
update: crate::table_catalog::NamespacePropertiesUpdate,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::NamespacePropertiesUpdateResult> {
let mut namespaces = self.namespaces.lock().await;
let entry = namespaces
.iter_mut()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.ok_or_else(|| {
crate::table_catalog::TableCatalogStoreError::NotFound(format!("namespace {table_bucket}/{namespace}"))
})?;
Ok(update.apply_to(entry))
}
async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.namespaces
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace));
Ok(())
}
async fn create_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
self.tables.lock().await.push(entry);
Ok(())
}
async fn register_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
if let Some(pause) = &self.register_table_pause {
pause.started.notify_one();
pause.release.notified().await;
}
self.tables.lock().await.push(entry);
Ok(())
}
async fn register_table_with_publication(
&self,
entry: crate::table_catalog::TableEntry,
publication: &(dyn crate::table_catalog::TableCommitPublication + Sync),
) -> crate::table_catalog::TableCatalogStoreResult<()> {
publication.begin_table_bucket(&entry.table_bucket).await?;
if !publication.holds_table_bucket(&entry.table_bucket) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table registration requires a table-bucket publication fence".to_string(),
));
}
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
publication
.prepare(&entry.table_bucket, &entry.namespace, &entry.table)
.await?;
if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table registration requires a table publication fence".to_string(),
));
}
self.register_table(entry).await
}
async fn list_tables(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned()
.collect())
}
async fn list_all_tables(
&self,
table_bucket: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket)
.cloned()
.collect())
}
async fn load_table(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::TableEntry>> {
Ok(self
.tables
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)
.cloned())
}
async fn commit_table(
&self,
request: crate::table_catalog::TableCommitRequest,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::TableCommitResult> {
let mut tables = self.tables.lock().await;
let Some(index) = tables.iter().position(|entry| {
entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table
}) else {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table {}/{}/{}",
request.table_bucket, request.namespace, request.table
)));
};
let current = tables[index].clone();
if current.version_token != request.expected_version_token {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current table version token does not match expected token".to_string(),
));
}
if current.metadata_location != request.expected_metadata_location {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current table metadata location does not match expected location".to_string(),
));
}
if let Some(pause) = &self.commit_table_pause {
pause.started.notify_one();
pause.release.notified().await;
}
let mut next = current.clone();
next.metadata_location = request.new_metadata_location.clone();
next.version_token = "token-committed".to_string();
next.generation = next.generation.saturating_add(1);
tables[index] = next.clone();
drop(tables);
let commit_log = crate::table_catalog::CommitLogEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
commit_id: request.commit_id,
idempotency_key: request.idempotency_key,
table_id: current.table_id,
operation: request.operation,
expected_version_token: request.expected_version_token,
new_version_token: next.version_token.clone(),
previous_metadata_location: request.expected_metadata_location,
new_metadata_location: request.new_metadata_location,
requirements: request.requirements,
status: crate::table_catalog::CommitLogStatus::Committed,
writer: request.writer,
created_at: None,
updated_at: None,
};
self.commits.lock().await.push(commit_log.clone());
Ok(crate::table_catalog::TableCommitResult { table: next, commit_log })
}
async fn commit_table_with_publication(
&self,
request: crate::table_catalog::TableCommitRequest,
publication: &(dyn crate::table_catalog::TableCommitPublication + Sync),
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::TableCommitResult> {
publication
.prepare(&request.table_bucket, &request.namespace, &request.table)
.await?;
if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) {
return Err(crate::table_catalog::TableCatalogStoreError::Internal(
"table commit requires a table publication fence".to_string(),
));
}
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
self.commit_table(request).await
}
async fn drop_table(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.tables
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table));
Ok(())
}
async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> {
if self.get_table_bucket(&entry.table_bucket).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"table bucket {}",
entry.table_bucket
)));
}
if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"namespace {}/{}",
entry.table_bucket, entry.namespace
)));
}
self.views.lock().await.push(entry);
Ok(())
}
async fn list_views(
&self,
table_bucket: &str,
namespace: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Vec<crate::table_catalog::ViewEntry>> {
Ok(self
.views
.lock()
.await
.iter()
.filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace)
.cloned()
.collect())
}
async fn load_view(
&self,
table_bucket: &str,
namespace: &str,
view: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::ViewEntry>> {
Ok(self
.views
.lock()
.await
.iter()
.find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)
.cloned())
}
async fn replace_view(
&self,
request: crate::table_catalog::ViewCommitRequest,
) -> crate::table_catalog::TableCatalogStoreResult<crate::table_catalog::ViewCommitResult> {
let mut views = self.views.lock().await;
let Some(index) = views.iter().position(|entry| {
entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view
}) else {
return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!(
"view {}/{}/{}",
request.table_bucket, request.namespace, request.view
)));
};
let current = views[index].clone();
if current.version_token != request.expected_version_token {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current view version token does not match expected token".to_string(),
));
}
if current.metadata_location != request.expected_metadata_location {
return Err(crate::table_catalog::TableCatalogStoreError::Conflict(
"current view metadata location does not match expected location".to_string(),
));
}
let mut next = current;
next.metadata_location = request.new_metadata_location;
next.version_token = "token-view-committed".to_string();
next.generation = next.generation.saturating_add(1);
views[index] = next.clone();
Ok(crate::table_catalog::ViewCommitResult { view: next })
}
async fn drop_view(
&self,
table_bucket: &str,
namespace: &str,
view: &str,
) -> crate::table_catalog::TableCatalogStoreResult<()> {
self.views
.lock()
.await
.retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view));
Ok(())
}
async fn get_commit_by_id(
&self,
_table_bucket: &str,
_table_id: &str,
_commit_id: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::CommitLogEntry>> {
Ok(None)
}
async fn get_commit_by_idempotency_key(
&self,
_table_bucket: &str,
_table_id: &str,
_idempotency_key: &str,
) -> crate::table_catalog::TableCatalogStoreResult<Option<crate::table_catalog::CommitLogEntry>> {
Ok(None)
}
}
+195 -3
View File
@@ -3,9 +3,7 @@ use super::identifier::{
default_table_lifecycle_path, default_table_marker_path, default_table_root_prefix, is_valid_table_metadata_file_name, default_table_lifecycle_path, default_table_marker_path, default_table_root_prefix, is_valid_table_metadata_file_name,
namespace_name_from_marker_path, table_name_from_marker_path, validate_object_mutation, namespace_name_from_marker_path, table_name_from_marker_path, validate_object_mutation,
}; };
use super::test_support::{ use super::test_support::{BlockingObjectPublication, TestCatalogObjectBackend, UnserializedTestPublication};
BlockingObjectPublication, NoopTableCatalogStore, TestCatalogObjectBackend, UnserializedTestPublication,
};
use super::*; use super::*;
use datafusion::{ use datafusion::{
arrow::{ arrow::{
@@ -224,6 +222,200 @@ fn catalog_object_listing_rejects_missing_or_stalled_continuation_tokens() {
); );
} }
struct NoopTableCatalogStore;
#[async_trait::async_trait]
impl TableCatalogStore for NoopTableCatalogStore {
async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<Option<TableBucketEntry>> {
Ok(None)
}
async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult<Vec<NamespaceEntry>> {
Ok(Vec::new())
}
async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Option<NamespaceEntry>> {
Ok(None)
}
async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn register_table_with_publication(
&self,
entry: TableEntry,
publication: &(dyn TableCommitPublication + Sync),
) -> TableCatalogStoreResult<()> {
publication.begin_table_bucket(&entry.table_bucket).await?;
if !publication.holds_table_bucket(&entry.table_bucket) {
return Err(TableCatalogStoreError::Internal(
"table registration requires a table-bucket publication fence".to_string(),
));
}
let _publication_completion = TableCommitPublicationCompletion::new(publication);
publication
.prepare(&entry.table_bucket, &entry.namespace, &entry.table)
.await?;
if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) {
return Err(TableCatalogStoreError::Internal(
"table registration requires a table publication fence".to_string(),
));
}
self.register_table(entry).await
}
async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
Ok(Vec::new())
}
async fn list_all_tables(&self, _table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
Ok(Vec::new())
}
async fn load_table(
&self,
_table_bucket: &str,
_namespace: &str,
_table: &str,
) -> TableCatalogStoreResult<Option<TableEntry>> {
Ok(None)
}
async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult<TableCommitResult> {
let table = TableEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: request.table_bucket,
namespace: request.namespace,
table: request.table,
table_id: "table-id".to_string(),
table_uuid: "table-uuid".to_string(),
format: "ICEBERG".to_string(),
format_version: 2,
warehouse_location: "s3://analytics/tables/table-id".to_string(),
metadata_location: request.new_metadata_location.clone(),
version_token: "token-v2".to_string(),
generation: 2,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
};
let commit_log = CommitLogEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
commit_id: request.commit_id,
idempotency_key: request.idempotency_key,
table_id: table.table_id.clone(),
operation: request.operation,
expected_version_token: request.expected_version_token,
new_version_token: table.version_token.clone(),
previous_metadata_location: request.expected_metadata_location,
new_metadata_location: table.metadata_location.clone(),
requirements: request.requirements,
status: CommitLogStatus::Committed,
writer: request.writer,
created_at: None,
updated_at: None,
};
Ok(TableCommitResult { table, commit_log })
}
async fn commit_table_with_publication(
&self,
request: TableCommitRequest,
publication: &(dyn TableCommitPublication + Sync),
) -> TableCatalogStoreResult<TableCommitResult> {
publication
.prepare(&request.table_bucket, &request.namespace, &request.table)
.await?;
if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) {
return Err(TableCatalogStoreError::Internal(
"table commit requires a table publication fence".to_string(),
));
}
let _publication_completion = TableCommitPublicationCompletion::new(publication);
self.commit_table(request).await
}
async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<Vec<ViewEntry>> {
Ok(Vec::new())
}
async fn load_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<Option<ViewEntry>> {
Ok(None)
}
async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult<ViewCommitResult> {
Ok(ViewCommitResult {
view: ViewEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: request.table_bucket,
namespace: request.namespace,
view: request.view,
view_id: "view-id".to_string(),
view_uuid: "view-uuid".to_string(),
format: "ICEBERG_VIEW".to_string(),
format_version: 1,
warehouse_location: "s3://analytics/views/view-id".to_string(),
metadata_location: request.new_metadata_location,
version_token: "token-v2".to_string(),
generation: 2,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
},
})
}
async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn get_commit_by_id(
&self,
_table_bucket: &str,
_table_id: &str,
_commit_id: &str,
) -> TableCatalogStoreResult<Option<CommitLogEntry>> {
Ok(None)
}
async fn get_commit_by_idempotency_key(
&self,
_table_bucket: &str,
_table_id: &str,
_idempotency_key: &str,
) -> TableCatalogStoreResult<Option<CommitLogEntry>> {
Ok(None)
}
}
#[tokio::test] #[tokio::test]
async fn table_catalog_store_trait_covers_entry_read_write_shapes() { async fn table_catalog_store_trait_covers_entry_read_write_shapes() {
let store: &dyn TableCatalogStore = &NoopTableCatalogStore; let store: &dyn TableCatalogStore = &NoopTableCatalogStore;
@@ -4033,7 +4033,7 @@ if [[ -s "$ECSTORE_REMOTE_TIER_DELETE_STATE_BYPASS_HITS_FILE" ]]; then
report_failure "remote tier delete state access must stay behind ECStore tier sweeper owner helpers: $(paste -sd '; ' "$ECSTORE_REMOTE_TIER_DELETE_STATE_BYPASS_HITS_FILE")" report_failure "remote tier delete state access must stay behind ECStore tier sweeper owner helpers: $(paste -sd '; ' "$ECSTORE_REMOTE_TIER_DELETE_STATE_BYPASS_HITS_FILE")"
fi fi
RUSTFS_OWNER_LOCAL_STATIC_NAMES='(KEYSTONE_AUTH|KEYSTONE_MAPPER|KEYSTONE_CONFIG|LICENSE_STATE|LICENSE_VERIFIER|CPU_CONT_GUARD|PROFILING_CANCEL_TOKEN|MEMORY_SYSTEM|DIAL9_TELEMETRY_GUARD|DISPLAY_CONFIG_SNAPSHOT|GLOBAL_CONFIG_SNAPSHOT|BUFFER_CONFIG_SINGLETON|BUFFER_PROFILE_ENABLED|LEGACY_CREDENTIAL_WARNED_KEYS|CONSOLE_CONFIG|ACTIVE_HTTP_REQUESTS|USE_STARSHARD_CACHE|BUCKET_CACHE_SMALL|BUCKET_CACHE_LARGE|GLOBAL_SSE_DEK_PROVIDER|SSE_TEST_LOCK|AUTH_FS|LOCK_STATS|DEADLOCK_DETECTOR|GET_OBJECT_BUFFER_THRESHOLD_WARNED|GET_READER_STREAM_BUFFER_SIZE_OVERRIDE|OBJECT_SEEK_SUPPORT_THRESHOLD|OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS|SUPPORTED_HEADERS|SITE_REPLICATION_PEER_CLIENT|AUDIT_MODULE_ENABLED|NOTIFY_MODULE_ENABLED|PERSISTED_NOTIFY_MODULE_ENABLED|PERSISTED_AUDIT_MODULE_ENABLED|PERSISTED_MODULE_SWITCH_CONFIGURED|DELETE_TAIL_TOTAL|DELETE_CLEANUP_TOTAL|DELETE_REPLICATION_TOTAL|DELETE_NOTIFY_TOTAL|EMBEDDED_SERVER_STARTED|TEST_OUTBOUND_TLS_GENERATION|TEST_REMAINING_FAILURES|CAPACITY_DIRTY_SCOPE_ENV|CAPACITY_DIRTY_SCOPE_INIT|GLOBAL_ENV)' RUSTFS_OWNER_LOCAL_STATIC_NAMES='(KEYSTONE_AUTH|KEYSTONE_MAPPER|KEYSTONE_CONFIG|LICENSE_STATE|LICENSE_VERIFIER|CPU_CONT_GUARD|PROFILING_CANCEL_TOKEN|MEMORY_SYSTEM|DIAL9_TELEMETRY_GUARD|DISPLAY_CONFIG_SNAPSHOT|GLOBAL_CONFIG_SNAPSHOT|BUFFER_CONFIG_SINGLETON|BUFFER_PROFILE_ENABLED|LEGACY_CREDENTIAL_WARNED_KEYS|CONSOLE_CONFIG|ACTIVE_HTTP_REQUESTS|USE_STARSHARD_CACHE|BUCKET_CACHE_SMALL|BUCKET_CACHE_LARGE|GLOBAL_SSE_DEK_PROVIDER|SSE_TEST_LOCK|AUTH_FS|LOCK_STATS|DEADLOCK_DETECTOR|GET_OBJECT_BUFFER_THRESHOLD_WARNED|GET_READER_STREAM_BUFFER_SIZE_OVERRIDE|OBJECT_SEEK_SUPPORT_THRESHOLD|OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS|SUPPORTED_HEADERS|SITE_REPLICATION_PEER_CLIENT|SITE_REPLICATION_STATE_LOCK|AUDIT_MODULE_ENABLED|NOTIFY_MODULE_ENABLED|PERSISTED_NOTIFY_MODULE_ENABLED|PERSISTED_AUDIT_MODULE_ENABLED|PERSISTED_MODULE_SWITCH_CONFIGURED|DELETE_TAIL_TOTAL|DELETE_CLEANUP_TOTAL|DELETE_REPLICATION_TOTAL|DELETE_NOTIFY_TOTAL|EMBEDDED_SERVER_STARTED|TEST_OUTBOUND_TLS_GENERATION|TEST_REMAINING_FAILURES|CAPACITY_DIRTY_SCOPE_ENV|CAPACITY_DIRTY_SCOPE_INIT|GLOBAL_ENV)'
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
+1 -3
View File
@@ -984,9 +984,7 @@ trace_hot_spans=(
"crates/ecstore/src/store/object.rs:handle_get_object_info" "crates/ecstore/src/store/object.rs:handle_get_object_info"
"crates/ecstore/src/set_disk/ops/object.rs:get_object_info" "crates/ecstore/src/set_disk/ops/object.rs:get_object_info"
"crates/ecstore/src/store/mod.rs:list_objects_v2" "crates/ecstore/src/store/mod.rs:list_objects_v2"
# The ECStore handle_list_objects_v2 forwarder was folded into the trait impl "crates/ecstore/src/store/list.rs:handle_list_objects_v2"
# above, so store/mod.rs now carries this hot path's TRACE requirement
# directly (backlog#1821).
"crates/ecstore/src/core/sets.rs:list_objects_v2" "crates/ecstore/src/core/sets.rs:list_objects_v2"
"crates/ecstore/src/set_disk/ops/list.rs:list_objects_v2" "crates/ecstore/src/set_disk/ops/list.rs:list_objects_v2"
"rustfs/src/app/bucket_usecase.rs:execute_list_objects_v2" "rustfs/src/app/bucket_usecase.rs:execute_list_objects_v2"
+1 -1
View File
@@ -26,7 +26,7 @@ cd "$(dirname "$0")/.."
# Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s # Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s
# to verify S3 behavior and does not widen the production s3s surface. # to verify S3 behavior and does not widen the production s3s surface.
S3S_IMPORT_FILES_BASELINE=213 S3S_IMPORT_FILES_BASELINE=213
S3_ERROR_LINES_BASELINE=1620 S3_ERROR_LINES_BASELINE=1621
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
E2E_TEST_GLOB='--glob=!crates/e2e_test/**' E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
-1
View File
@@ -209,7 +209,6 @@ export RUSTFS_NS_SCANNER_INTERVAL=60 # Object scanning interval in seconds
# Storage level compression (compression at object storage level) # Storage level compression (compression at object storage level)
# export RUSTFS_COMPRESSION_ENABLED=true # Whether to enable storage-level compression for objects # export RUSTFS_COMPRESSION_ENABLED=true # Whether to enable storage-level compression for objects
# export RUSTFS_COMPRESSION_MULTIPART_ENABLED=true # Additionally compress multipart uploads (staged rollout switch: enable only after the whole fleet runs a build with the resumable decompressor; see docs/architecture/compat-cleanup-register.md)
# HTTP Response Compression (whitelist-based, aligned with MinIO) # HTTP Response Compression (whitelist-based, aligned with MinIO)
# By default, HTTP response compression is DISABLED (aligned with MinIO behavior) # By default, HTTP response compression is DISABLED (aligned with MinIO behavior)
@@ -195,13 +195,6 @@ IFS= read -r -d '' expected_docker_automatic_guard <<'EOF' || true
EOF EOF
expected_docker_automatic_guard=${expected_docker_automatic_guard%$'\n'} expected_docker_automatic_guard=${expected_docker_automatic_guard%$'\n'}
require_job_if "$docker_workflow" "build-check" "$expected_docker_automatic_guard" require_job_if "$docker_workflow" "build-check" "$expected_docker_automatic_guard"
require_line "$docker_workflow" ' source_ref: ${{ steps.check.outputs.source_ref }}' "Docker source ref output"
require_line "$docker_workflow" ' source_ref="$HEAD_SHA"' "automatic Docker source ref"
require_line "$docker_workflow" ' source_ref="$tag_ref"' "manual Docker source ref"
require_line "$docker_workflow" ' ref: ${{ needs.build-check.outputs.source_ref }}' "Docker release source checkout"
require_line "$docker_workflow" ' SOURCE_REVISION="$(git rev-parse HEAD)"' "Docker source revision resolution"
require_line "$docker_workflow" ' LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"' "Docker revision label"
require_absent "$docker_workflow" 'org.opencontainers.image.revision=${{ github.sha }}' "Docker revision must not use the workflow branch SHA"
docker_manual_guard=$(awk ' docker_manual_guard=$(awk '
$0 == " *-preview*)" { in_preview = 1 } $0 == " *-preview*)" { in_preview = 1 }