mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 18:08:21 +00:00
Compare commits
187 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ea40a41c9b | |||
| 33eff4c3c4 | |||
| a2f16aa066 | |||
| 4c8b9f87e1 | |||
| 3272730c13 | |||
| 1862112d0c | |||
| cd0ac02879 | |||
| 6cf9cf7bb5 | |||
| f1f86ee9d0 | |||
| 1eef0de003 | |||
| a118d7e4fd | |||
| ed1bedf1fb | |||
| 4392f94e1a | |||
| 81d7b7d07a | |||
| e26668e62c | |||
| 8d3511c1b3 | |||
| d172d05e86 | |||
| 0d86c50760 | |||
| 526d6f667e | |||
| dcf3e4b9e8 | |||
| 04b9c8fd36 | |||
| c1f66969d7 | |||
| cfa9276fad | |||
| db8f55cb97 | |||
| 7f23a1ba91 | |||
| 1619c4be60 | |||
| 72fd7339c9 | |||
| 71e83aeec4 | |||
| 9138c24571 | |||
| e9f5318027 | |||
| 69e8ef9af5 | |||
| 0b2a46b36f | |||
| 56509ead1f | |||
| ffe889ad59 | |||
| e11ce2f132 | |||
| eca6bc1600 | |||
| 85be26b3c1 | |||
| ebbcfa3ac2 | |||
| ebd0531124 | |||
| 4421d4829f | |||
| d6c62b9601 | |||
| d91086d094 | |||
| 69719c257e | |||
| 67a19021b5 | |||
| 0ff3d4cbf4 | |||
| 6f29431a65 | |||
| 5a4c063d16 | |||
| e2be34cade | |||
| d60a77b750 | |||
| 307f50ee1b | |||
| c48a6330d0 | |||
| 8ac2ff5c61 | |||
| eb41f45175 | |||
| 122d200675 | |||
| 161e515c72 | |||
| 83cf063b45 | |||
| f8bbfcbeb1 | |||
| 710dcb4865 | |||
| f5cced910a | |||
| 8c9249054f | |||
| 7c2b513613 | |||
| 00844721ff | |||
| 068a0c2b8c | |||
| 5b54c4303d | |||
| 6178083985 | |||
| e16c07b9cd | |||
| 1ac28d6459 | |||
| 9b66040a02 | |||
| 7710f70fda | |||
| aa4d3317ed | |||
| f704d015d6 | |||
| 6b86d44cac | |||
| e3c15f012c | |||
| d2b1003612 | |||
| 36deab8670 | |||
| e11fcfbd08 | |||
| 11eecdc888 | |||
| 80eb4244a3 | |||
| e4da9bd718 | |||
| e28430ab3d | |||
| db4707f187 | |||
| 3a0dbccc2e | |||
| 846517625b | |||
| f21e88b112 | |||
| a5594c3d89 | |||
| fc927caadd | |||
| b7e6334c13 | |||
| 65091aa6a8 | |||
| 3c78a56ab0 | |||
| bdd7ecd205 | |||
| a70a3787d8 | |||
| b2ae430805 | |||
| 299eb0d965 | |||
| f5a780099b | |||
| 5cfafcf39b | |||
| a49243c671 | |||
| c2a15f5214 | |||
| ee54f1e618 | |||
| e6b85b60a8 | |||
| 8c1e3c09ff | |||
| ca06c7ec2c | |||
| 4a41325d1a | |||
| 45e2bd0c28 | |||
| e2fb0427f9 | |||
| a825326ede | |||
| e313276e49 | |||
| 66af487978 | |||
| d668a9293f | |||
| ace28c1f85 | |||
| 2ad8ab534e | |||
| f7df4fa62a | |||
| ca4e66daab | |||
| 5b9c5289c2 | |||
| 3f9b84ec70 | |||
| 73bd5d9d95 | |||
| 398d2d87c8 | |||
| 59d8d93832 | |||
| 59494d5089 | |||
| 019e80a218 | |||
| 9546baf1ab | |||
| 60d8e8a20b | |||
| 24ca61eb6e | |||
| d92c563b9e | |||
| e087044658 | |||
| 16d381fc0e | |||
| 1021d7228a | |||
| 0a246e3736 | |||
| 380ed40b47 | |||
| 679ea238de | |||
| baadaccc30 | |||
| 87d47a6e5d | |||
| fba0b34f19 | |||
| c9eeb2fa8a | |||
| 2f83d6789b | |||
| 7a4a3d27c6 | |||
| 4c5e73b2f2 | |||
| 4c44bc649a | |||
| 3b49842df0 | |||
| 848b330825 | |||
| 270a003c55 | |||
| 8b57076194 | |||
| d31bd3cd10 | |||
| 698ebdfb3f | |||
| c7233d6624 | |||
| 493a2cc1ba | |||
| 3ebb426abe | |||
| 5aac224a97 | |||
| 1b6ae33ce0 | |||
| 537d34b8cd | |||
| 3fdf2964c8 | |||
| 2e5874f839 | |||
| 5d05897ae0 | |||
| 6850482247 | |||
| b00b7ab8f1 | |||
| 924958bab5 | |||
| 968ec4a8be | |||
| 8d34b4d101 | |||
| 882ad4c113 | |||
| e9728192e2 | |||
| a206a0779e | |||
| 6cce3d60bb | |||
| 42433584ab | |||
| ba6a0f25d9 | |||
| 5e3010c6b5 | |||
| bc888931fd | |||
| 4ac7c56c89 | |||
| ddacce6e75 | |||
| 31cb720471 | |||
| 603bdea516 | |||
| a076ae4045 | |||
| 8a8be12f0b | |||
| 2ecf6b4575 | |||
| 2aa0148454 | |||
| 3289d40ce9 | |||
| 849837e262 | |||
| 727a10e111 | |||
| 3747d19ce5 | |||
| 1148e76279 | |||
| 320b788a50 | |||
| 3c31eaf06f | |||
| fe2516ee86 | |||
| 7ca69eb39c | |||
| 95627cb601 | |||
| d97e059c3c | |||
| d900e11a09 | |||
| f1ff9a36bc | |||
| 276eea1fba |
+14
-7
@@ -34,7 +34,8 @@ e2e-vault = { max-threads = 1 }
|
||||
|
||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13). They
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13, and
|
||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||
@@ -90,7 +91,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# e2e-reliability test-group note above). The matching ci-profile override is at
|
||||
# the end of the file, after [profile.ci] is declared.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.default.overrides]]
|
||||
@@ -155,7 +156,7 @@ retries = 2
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios under the ci profile too
|
||||
@@ -218,7 +219,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 36 nightly = 56 total
|
||||
# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
@@ -251,10 +252,16 @@ test-group = 'ecstore-serial-flaky'
|
||||
# cluster, so it keeps the lane's parallel-safe / no-external-dependency
|
||||
# properties. The RustFS warm backend has no loopback guard (that guard is
|
||||
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
|
||||
#
|
||||
# Disk compression (backlog#1848): the `compression` module joins the smoke
|
||||
# lane so the multipart disk-compression roundtrips (restored after
|
||||
# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate.
|
||||
# Single-node servers on random ports with isolated temp dirs — meets the
|
||||
# admission criteria unchanged.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
@@ -344,7 +351,7 @@ path = "junit.xml"
|
||||
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||
# nightly 4-node lane.
|
||||
# * replication_extension_test — repl-1 already splits it into the PR
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (27 slow) lanes and reserves
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
|
||||
# it for those, so e2e-full does not double-run it.
|
||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||
@@ -383,7 +390,7 @@ path = "junit.xml"
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
|
||||
@@ -17,9 +17,11 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Metric source: the KMS operation-policy choke point in
|
||||
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||
# key material, and tokens never appear in labels.
|
||||
# crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the
|
||||
# label-less key-lifecycle gauge published by the deletion worker's sweep
|
||||
# (crates/kms/src/deletion_worker.rs). All label values are bounded static
|
||||
# strings (operation, op_class, outcome, error_class, backend, scope); key
|
||||
# identifiers, key material, and tokens never appear in labels.
|
||||
#
|
||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||
#
|
||||
@@ -212,3 +214,38 @@ groups:
|
||||
circuit_open until the half-open probe succeeds or returns
|
||||
a non-retryable failure.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 7. KmsKeyRotationOverdue
|
||||
# The least recently rotated usable key has gone more than 400
|
||||
# days without a rotation (measured from creation for keys with
|
||||
# no recorded rotation). Direct gauge state published by the
|
||||
# deletion worker's sweep, so no traffic guard applies; the
|
||||
# one-hour hold only bridges scrape gaps. The worker runs only
|
||||
# on backends with the schedule_deletion capability, so on the
|
||||
# Static backend the series never exists and this alert cannot
|
||||
# fire — that backend cannot rotate either; see the rotation
|
||||
# driver matrix in docs/operations/kms-backend-security.md.
|
||||
# Threshold: 400 days — conservative default sitting above a
|
||||
# one-year rotation policy. Align it with the rotation period
|
||||
# your compliance policy requires, and with
|
||||
# RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due
|
||||
# verdict and this aggregate alert agree.
|
||||
# ------------------------------------------------------------------
|
||||
- alert: KmsKeyRotationOverdue
|
||||
expr: |
|
||||
rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
component: kms
|
||||
annotations:
|
||||
summary: "Oldest KMS key unrotated for more than 400 days"
|
||||
description: >-
|
||||
The least recently rotated usable KMS key was last rotated
|
||||
{{ $value | humanizeDuration }} ago (measured from creation
|
||||
for keys with no recorded rotation). List keys through the
|
||||
admin API and read rotation_due / rotation_due_reason for
|
||||
the per-key verdict; an "unsupported" reason means the
|
||||
backend cannot rotate at all.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"
|
||||
|
||||
@@ -85,7 +85,7 @@ runs:
|
||||
repo-token: ${{ github.token }}
|
||||
|
||||
- name: Install flatc
|
||||
uses: Nugine/setup-flatc@e7855e994773ce90094a3f1626d4afc9080c23ae # v1
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
|
||||
|
||||
@@ -182,7 +182,12 @@ jobs:
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
# Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2
|
||||
# (weekly schedule / manual dispatch only — dormant rio-v2 variant, see
|
||||
# rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md).
|
||||
# The second build below stays despite the reduced cadence: it warms the
|
||||
# rio-v2,e2e-test-hooks feature resolution the scheduled build restores,
|
||||
# which keeps that lane inside its 30-minute timeout.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
|
||||
@@ -533,7 +533,12 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary-rio-v2:
|
||||
name: Build RustFS Debug Binary (rio-v2)
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
||||
# default build, so this full-suite lane runs only on the weekly schedule
|
||||
# and manual dispatch. Per-PR cfg-seam coverage stays with
|
||||
# test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
|
||||
# docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
@@ -824,6 +829,9 @@ jobs:
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
# event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
|
||||
# with it (see the dormant-variant comment on that job).
|
||||
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
|
||||
@@ -94,6 +94,7 @@ jobs:
|
||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -118,6 +119,7 @@ jobs:
|
||||
short_sha=""
|
||||
is_prerelease=false
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
@@ -137,6 +139,7 @@ jobs:
|
||||
# Extract version info from commit message or use commit SHA
|
||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||
source_ref="$HEAD_SHA"
|
||||
|
||||
# Determine build type based on triggering workflow event and ref
|
||||
triggering_event="$TRIGGERING_EVENT"
|
||||
@@ -261,6 +264,23 @@ jobs:
|
||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||
tag_ref="refs/tags/$input_version"
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
if [[ "$input_version" == v* ]]; then
|
||||
tag_ref="refs/tags/${input_version#v}"
|
||||
else
|
||||
tag_ref="refs/tags/v$input_version"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
echo "❌ Release tag not found for Docker build: $input_version"
|
||||
exit 1
|
||||
fi
|
||||
source_ref="$tag_ref"
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
@@ -271,6 +291,7 @@ jobs:
|
||||
echo "short_sha=$short_sha"
|
||||
echo "is_prerelease=$is_prerelease"
|
||||
echo "create_latest=$create_latest"
|
||||
echo "source_ref=$source_ref"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker Build Summary:"
|
||||
@@ -281,6 +302,7 @@ jobs:
|
||||
echo " - Short SHA: $short_sha"
|
||||
echo " - Is prerelease: $is_prerelease"
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
@@ -308,6 +330,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
@@ -397,7 +420,8 @@ jobs:
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||
|
||||
@@ -55,3 +55,142 @@ jobs:
|
||||
|
||||
- name: Build RustFS
|
||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
# Vault Transit backends to every for_each_backend spec in
|
||||
# crates/kms/tests/behavior_*.rs (see crates/kms/AGENTS.md). rotate and
|
||||
# versioning are advertised only by the Vault backends, so without this lane
|
||||
# no CI run ever asserts the working half of behavior_rotation.rs — a
|
||||
# rotation that silently dropped historical key versions would stay green.
|
||||
# The same lane runs the dev-Vault #[ignore] tests and the two self-hosting
|
||||
# live scripts (AppRole login, three-node Raft leader failover).
|
||||
#
|
||||
# GitHub-hosted ubuntu-latest, deliberately not the self-hosted sm-standard
|
||||
# fleet: the HA failover script needs a working Docker daemon, and the
|
||||
# self-hosted fleet is heterogeneous — a docker-dependent workflow has been
|
||||
# burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
|
||||
kms-vault-lane:
|
||||
name: KMS live Vault lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Root token of the ephemeral loopback dev server. Not a secret: the
|
||||
# server lives only for this job, listens on 127.0.0.1, and holds only
|
||||
# keys the tests create. The literal value matters — the dev-Vault
|
||||
# #[ignore] fixtures in crates/kms/src/backends/vault.rs hardcode it.
|
||||
VAULT_LANE_TOKEN: dev-only-token
|
||||
VAULT_LANE_ADDR: http://127.0.0.1:8200
|
||||
# Keeps a runner-level proxy from swallowing the loopback dev-server
|
||||
# traffic (see crates/kms/AGENTS.md). Actions env keys are
|
||||
# case-insensitive, so only the uppercase form is set; reqwest reads
|
||||
# either casing.
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
# Dedicated key: rust-cache cannot tell runner images apart, so
|
||||
# sharing a key with an sm-standard lane would let two different
|
||||
# system images overwrite each other's artifacts (same reasoning as
|
||||
# ci.yml's ci-uring lane). Saved from this nightly job itself so the
|
||||
# next night starts warm.
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'true'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Install Vault CLI
|
||||
run: |
|
||||
set -euo pipefail
|
||||
wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list >/dev/null
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq vault
|
||||
vault version
|
||||
|
||||
- name: Start Vault dev server with KV2 and Transit engines
|
||||
run: |
|
||||
set -euo pipefail
|
||||
nohup vault server -dev \
|
||||
-dev-root-token-id="${VAULT_LANE_TOKEN}" \
|
||||
-dev-listen-address=127.0.0.1:8200 >/tmp/vault-dev.log 2>&1 &
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health"
|
||||
export VAULT_ADDR="${VAULT_LANE_ADDR}" VAULT_TOKEN="${VAULT_LANE_TOKEN}"
|
||||
# Dev mode mounts KV v2 at secret/ by default; Transit is explicit.
|
||||
# Prove both engines actually work rather than assuming the defaults.
|
||||
vault secrets enable transit
|
||||
vault kv put secret/rustfs-ci-lane-probe value=ok >/dev/null
|
||||
vault kv get secret/rustfs-ci-lane-probe >/dev/null
|
||||
vault write -f transit/keys/rustfs-ci-lane-probe >/dev/null
|
||||
|
||||
- name: Run rustfs-kms suite with the Vault lane on
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
run: cargo test -p rustfs-kms --locked
|
||||
|
||||
- name: Run dev-Vault ignored tests
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
# Filters select the dev-Vault-only #[ignore] tests. The AWS #[ignore]
|
||||
# tests (backends::aws, service_manager) stay excluded — they need real
|
||||
# AWS credentials and create billable keys. The AppRole and HA #[ignore]
|
||||
# tests are excluded here because their own scripts below provision the
|
||||
# Vault topology they need.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo test -p rustfs-kms --locked --lib backends::contract_tests -- --ignored
|
||||
cargo test -p rustfs-kms --locked --lib backends::vault -- --ignored
|
||||
cargo test -p rustfs-kms --locked --test vault_fault_injection -- --ignored
|
||||
|
||||
- name: Run AppRole live checks (self-hosting ephemeral Vault)
|
||||
run: bash scripts/test/vault_approle_kms_live.sh
|
||||
|
||||
- name: Show Vault dev server log on failure
|
||||
if: failure()
|
||||
run: tail -n 200 /tmp/vault-dev.log || true
|
||||
|
||||
# Three-node Raft leader failover (crates/kms/tests/vault_ha_failover_live.rs,
|
||||
# first validated by rustfs/rustfs#5653). Its own job so an election-timing
|
||||
# flake cannot mask the main lane's verdict, and vice versa. The script
|
||||
# provisions and tears down its own Docker cluster.
|
||||
kms-vault-ha-failover:
|
||||
name: KMS Vault HA failover lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
||||
run: bash scripts/test/vault_ha_kms_live.sh
|
||||
|
||||
@@ -322,6 +322,17 @@ jobs:
|
||||
sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install fpm
|
||||
|
||||
# Create config file for fpm (DEB build creates it in its package dir structure,
|
||||
# but fpm needs the file to exist before packaging)
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$VERSION" \
|
||||
@@ -362,6 +373,7 @@ jobs:
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
+68
-21
@@ -1,6 +1,6 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
> Last updated: 2026-07-02 · Revision: 2
|
||||
> Last updated: 2026-08-12 · Revision: 3
|
||||
>
|
||||
> This document describes the high-level architecture of RustFS.
|
||||
> If you want to familiarize yourself with the code base, you are in the right place!
|
||||
@@ -101,7 +101,10 @@ refactors.
|
||||
|
||||
The `rustfs` binary crate composes these libraries into the running server.
|
||||
`ecstore` remains the storage engine at the architectural center; its internal
|
||||
module split is tracked under `docs/architecture/`.
|
||||
module split is tracked under `docs/architecture/`. `rio-v2` is the
|
||||
feature-gated MinIO on-disk format compatibility I/O layer; it ships in no
|
||||
default build (lifecycle:
|
||||
[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)).
|
||||
|
||||
## Architecture Invariants
|
||||
|
||||
@@ -119,19 +122,44 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
||||
in one crate and re-exported or imported by others.
|
||||
- ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
|
||||
`BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
|
||||
- ⚠️ VIOLATED: `ReplicationStats` names three unrelated types
|
||||
(`crates/data-usage/src/data_usage.rs`,
|
||||
`crates/obs/src/metrics/collectors/replication.rs`,
|
||||
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
||||
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
||||
- `LastMinuteLatency` has two deliberately different implementations: the
|
||||
per-second bucketed accumulator in `crates/common/src/last_minute.rs` and
|
||||
the in-memory endpoint-health sample tracker in
|
||||
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
||||
why it stays local).
|
||||
- ✅ RESOLVED: `BackpressureConfig` and `DataUsageInfo` each have exactly one
|
||||
definition (`crates/io-core/src/backpressure.rs`,
|
||||
`crates/data-usage/src/data_usage.rs`). The zero-consumer
|
||||
`BackpressureSettings` copy that lingered in io-metrics was removed
|
||||
(rustfs/backlog#1833).
|
||||
|
||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||
storage-level abstractions (objects, buckets, disks, pools).
|
||||
- ⚠️ VIOLATED: 58 files under `crates/ecstore/src` reference `s3s`
|
||||
(`rg -l 's3s' crates/ecstore/src | wc -l`), `crates/ecstore/src/client/`
|
||||
is a ~9.4K-line embedded S3 HTTP client, and `crates/ecstore/Cargo.toml`
|
||||
depends on `s3s`, `http`, `hyper`/`hyper-util`/`hyper-rustls`, and
|
||||
`reqwest`. Target state: the engine's need to act as an S3 client
|
||||
(tiering, replication targets) is served by an extracted client crate,
|
||||
and ecstore holds no wire or DTO types.
|
||||
|
||||
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
||||
Individual crates should be testable in isolation.
|
||||
|
||||
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
||||
not bare `Error`).
|
||||
- ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
|
||||
`heal` use `anyhow` in library code.
|
||||
- ✅ RESOLVED (strategy): `snafu` is gone from source
|
||||
(`rg -l snafu crates/ rustfs/` is empty) and library code no longer uses
|
||||
`anyhow` (remaining hits are test code and the `e2e_test` crate; `heal`
|
||||
uses `thiserror`).
|
||||
- ⚠️ VIOLATED (naming): 6 crates still export a bare `pub enum Error`:
|
||||
`crypto`, `filemeta`, `heal`, `iam`, `policy`, and `replication`
|
||||
(`src/resync.rs`) — all `thiserror`-derived.
|
||||
|
||||
## Known Structural Issues
|
||||
|
||||
@@ -140,13 +168,25 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
### Critical
|
||||
|
||||
- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
|
||||
but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
|
||||
types instead of importing them.
|
||||
- **scanner/data-usage duplicate `.usage-cache.bin` serialization types.** The
|
||||
original finding ("common/scanner code duplication, ~3K lines") is resolved:
|
||||
`scanner` imports the shared data-usage types from `rustfs-data-usage` (see
|
||||
the `pub use rustfs_data_usage::…` re-exports at the top of
|
||||
`crates/scanner/src/data_usage_define.rs`). What remains: `scanner` and
|
||||
`data-usage` each hold their own serialization types for the scanner cache
|
||||
file (`DataUsageCacheInfo`/`DataUsageEntryInfo` in
|
||||
`crates/scanner/src/data_usage_define.rs` vs
|
||||
`DataUsageCacheInfo`/`DataUsageEntry` in
|
||||
`crates/data-usage/src/data_usage.rs`); convergence is tracked in
|
||||
rustfs/backlog#1828.
|
||||
|
||||
- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
|
||||
bucket management, erasure coding, replication, lifecycle, RPC, and configuration
|
||||
— all in one crate. It should be decomposed along its existing subdirectories.
|
||||
- **ecstore is a monolith (265 files, ~288K lines — roughly half is inline
|
||||
`#[cfg(test)]` code).** Measured with
|
||||
`find crates/ecstore/src -name '*.rs' | xargs wc -l`. It contains disk
|
||||
management, bucket management, erasure coding, replication, lifecycle, RPC,
|
||||
and configuration — all in one crate. It should be decomposed along its
|
||||
existing subdirectories; the split plan lives in
|
||||
[docs/architecture/ecstore-module-split-plan.md](docs/architecture/ecstore-module-split-plan.md).
|
||||
|
||||
### High
|
||||
|
||||
@@ -154,19 +194,26 @@ module split is tracked under `docs/architecture/`.
|
||||
`common → filemeta/madmin` edges must stay removed so leaf/helper crates do
|
||||
not regain upward dependencies.
|
||||
|
||||
- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. Storage policies now expose and consume
|
||||
explicit projections into the concurrency/io-core policy shapes, and workload
|
||||
- **Three-layer backpressure/deadlock policy bridging** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. The config types are no longer
|
||||
duplicated (`BackpressureConfig` and `DeadlockDetectorConfig` are each
|
||||
defined once, in io-core). Storage policies expose and consume explicit
|
||||
projections into the concurrency/io-core policy shapes, and workload
|
||||
admission snapshots are composed through provider registries; later work
|
||||
should use those bridges before deleting compatibility wrappers.
|
||||
|
||||
### Medium
|
||||
|
||||
- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
|
||||
mixed naming (bare `Error` vs descriptive names).
|
||||
- **Bare `Error` naming.** Error-handling strategy has converged on `thiserror`
|
||||
(no `snafu`, no `anyhow` in library code); the remaining inconsistency is the
|
||||
bare `pub enum Error` naming in the 6 crates listed under Invariant 6.
|
||||
|
||||
- **Ambiguous common vs utils boundary.** Both described as "utilities and data
|
||||
structures." Need clear ownership rules.
|
||||
- **`common` is mostly parked domain code, not shared utilities.** Of its
|
||||
6,724 lines, ~83% is scanner/heal domain code stranded there to break
|
||||
dependency cycles (`metrics.rs`, ~4,810 lines of scanner-domain metrics;
|
||||
`heal_channel.rs`, ~776 lines of heal-domain channel types). The
|
||||
"common vs utils" naming ambiguity is secondary to moving that code to its
|
||||
domain owners.
|
||||
|
||||
## Cross-Cutting Concerns
|
||||
|
||||
@@ -232,7 +279,7 @@ The binary (`main.rs`) boots in this order:
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ rustfs │ (binary + lib, 75K lines)
|
||||
│ rustfs │ (binary + lib)
|
||||
│ main │
|
||||
└────┬────┘
|
||||
│
|
||||
@@ -255,7 +302,7 @@ The binary (`main.rs`) boots in this order:
|
||||
│ │ │
|
||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (87K,core) │ │ (readers) │ │ (zero-copy) │
|
||||
│ (core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
|
||||
Generated
+244
-179
File diff suppressed because it is too large
Load Diff
+63
-62
@@ -41,7 +41,7 @@ members = [
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-rc.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -142,10 +142,10 @@ async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
futures-core = "0.3.33"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.33"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.8.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
@@ -154,7 +154,7 @@ hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
http-body-util = "0.1.5"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.2.0" }
|
||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -182,6 +182,7 @@ quick-xml = "0.41.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
@@ -230,9 +231,9 @@ aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
base64 = "0.23.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
@@ -268,7 +269,7 @@ lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15" }
|
||||
moka = { version = "0.12.16" }
|
||||
netif = "0.1.6"
|
||||
num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.12.1"
|
||||
@@ -294,7 +295,7 @@ serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
smallvec = { version = "1.15.2" }
|
||||
smartstring = "1.0.1"
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.2.2" }
|
||||
strum = { version = "0.28.0" }
|
||||
@@ -339,17 +340,17 @@ pyroscope = { version = "2.1.1" }
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.6" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.1", default-features = false }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use base64_simd::STANDARD;
|
||||
|
||||
|
||||
@@ -21,6 +21,13 @@ use crate::{
|
||||
Xxhash3, Xxhash64, Xxhash128,
|
||||
};
|
||||
|
||||
// DELIBERATE DUPLICATION of the x-amz-checksum-* names that also exist as
|
||||
// AMZ_CHECKSUM_* in rustfs-utils' headers module (crates/utils/src/http/
|
||||
// headers.rs): this crate is a zero-internal-dependency leaf, so it cannot
|
||||
// import them, and it additionally owns the RustFS extension names
|
||||
// (sha512/xxhash*) that utils does not carry. Values are pinned by the S3
|
||||
// wire protocol; do not merge without a maintainer decision on the leaf
|
||||
// boundary (backlog#1833).
|
||||
pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
|
||||
pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
|
||||
pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
|
||||
|
||||
@@ -41,6 +41,14 @@ pub const XXHASH_64_NAME: &str = "xxhash64";
|
||||
pub const XXHASH_128_NAME: &str = "xxhash128";
|
||||
pub const MD5_NAME: &str = "md5";
|
||||
|
||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
||||
/// this enum owns the **streaming-hash algorithm registry**, including the
|
||||
/// RustFS extensions (sha512, xxhash3/64/128). The on-disk xl.meta bitset
|
||||
/// lives in `rustfs_rio::ChecksumType` (crates/rio/src/checksum.rs, varint
|
||||
/// bits are append-only), and the MinIO-port client keeps its own
|
||||
/// `ChecksumMode` (crates/ecstore/src/client/checksum.rs). When adding an
|
||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
||||
/// each other.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
#[non_exhaustive]
|
||||
pub enum ChecksumAlgorithm {
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::last_minute::{self};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub struct ReplicationLatency {
|
||||
// Delays for single and multipart PUT requests
|
||||
upload_histogram: last_minute::LastMinuteHistogram,
|
||||
}
|
||||
|
||||
impl ReplicationLatency {
|
||||
// Merge two ReplicationLatency
|
||||
pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
|
||||
self.upload_histogram.merge(&other.upload_histogram);
|
||||
self
|
||||
}
|
||||
|
||||
// Get upload delay (categorized by object size interval)
|
||||
pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
|
||||
let mut ret = HashMap::new();
|
||||
let avg = self.upload_histogram.get_avg_data();
|
||||
for (i, v) in avg.iter().enumerate() {
|
||||
let avg_duration = v.avg();
|
||||
ret.insert(self.size_tag_to_string(i), avg_duration.as_millis() as u64);
|
||||
}
|
||||
ret
|
||||
}
|
||||
pub fn update(&mut self, size: i64, during: std::time::Duration) {
|
||||
self.upload_histogram.add(size, during);
|
||||
}
|
||||
|
||||
// Simulate the conversion from size tag to string
|
||||
fn size_tag_to_string(&self, tag: usize) -> String {
|
||||
match tag {
|
||||
0 => String::from("Size < 1 KiB"),
|
||||
1 => String::from("Size < 1 MiB"),
|
||||
2 => String::from("Size < 10 MiB"),
|
||||
3 => String::from("Size < 100 MiB"),
|
||||
4 => String::from("Size < 1 GiB"),
|
||||
_ => String::from("Size > 1 GiB"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone, Default)]
|
||||
// pub struct ReplicationLastMinute {
|
||||
// pub last_minute: LastMinuteLatency,
|
||||
// }
|
||||
|
||||
// impl ReplicationLastMinute {
|
||||
// pub fn merge(&mut self, other: ReplicationLastMinute) -> ReplicationLastMinute {
|
||||
// let mut nl = ReplicationLastMinute::default();
|
||||
// nl.last_minute = self.last_minute.merge(&mut other.last_minute);
|
||||
// nl
|
||||
// }
|
||||
|
||||
// pub fn add_size(&mut self, n: i64) {
|
||||
// let t = SystemTime::now()
|
||||
// .duration_since(UNIX_EPOCH)
|
||||
// .expect("Time went backwards")
|
||||
// .as_secs();
|
||||
// self.last_minute.add_all(t - 1, &AccElem { total: t - 1, size: n as u64, n: 1 });
|
||||
// }
|
||||
|
||||
// pub fn get_total(&self) -> AccElem {
|
||||
// self.last_minute.get_total()
|
||||
// }
|
||||
// }
|
||||
|
||||
// impl fmt::Display for ReplicationLastMinute {
|
||||
// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
// let t = self.last_minute.get_total();
|
||||
// write!(f, "ReplicationLastMinute sz= {}, n= {}, dur= {}", t.size, t.n, t.total)
|
||||
// }
|
||||
// }
|
||||
@@ -572,44 +572,3 @@ mod tests {
|
||||
assert_eq!(total.n, 6);
|
||||
}
|
||||
}
|
||||
|
||||
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LastMinuteHistogram {
|
||||
histogram: Vec<LastMinuteLatency>,
|
||||
size: u32,
|
||||
}
|
||||
|
||||
impl LastMinuteHistogram {
|
||||
pub fn merge(&mut self, other: &LastMinuteHistogram) {
|
||||
for i in 0..self.histogram.len() {
|
||||
self.histogram[i].merge(&other.histogram[i]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(&mut self, size: i64, t: Duration) {
|
||||
let index = size_to_tag(size);
|
||||
self.histogram[index].add(&t);
|
||||
}
|
||||
|
||||
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
|
||||
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
|
||||
for (i, elem) in self.histogram.iter_mut().enumerate() {
|
||||
res[i] = elem.get_total();
|
||||
}
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
fn size_to_tag(size: i64) -> usize {
|
||||
match size {
|
||||
_ if size < 1024 => 0, // sizeLessThan1KiB
|
||||
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
|
||||
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
|
||||
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
|
||||
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
|
||||
_ => 5, // sizeGreaterThan1GiB
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,13 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_stats;
|
||||
// pub mod error;
|
||||
pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -915,11 +915,13 @@ const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
|
||||
const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
|
||||
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
|
||||
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
|
||||
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
|
||||
const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum ScanCyclePartialReason {
|
||||
@@ -1424,6 +1426,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
|
||||
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
|
||||
SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
|
||||
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
||||
}
|
||||
}
|
||||
@@ -1752,6 +1755,11 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_deferred(duration: Duration) {
|
||||
global_metrics().record_scan_cycle_deferred(duration);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
@@ -2549,6 +2557,17 @@ impl Metrics {
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
self.record_scan_cycle_partial_with_source(duration, reason, None);
|
||||
}
|
||||
@@ -4264,6 +4283,21 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_deferred_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_deferred(Duration::from_millis(250));
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
|
||||
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
|
||||
assert_eq!(report.last_cycle_duration_seconds, 0.25);
|
||||
assert_eq!(report.failed_cycles, 0);
|
||||
assert_eq!(report.superseded_cycles, 0);
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#![allow(clippy::all)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@@ -13,13 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub struct TargetID {
|
||||
id: String,
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl TargetID {
|
||||
fn to_string(&self) -> String {
|
||||
format!("{}:{}", self.id, self.name)
|
||||
}
|
||||
}
|
||||
/// Cross-crate lock identity used to fence table-bucket publication against
|
||||
/// object mutations that bypass the S3 request authorization layer.
|
||||
pub const TABLE_BUCKET_PUBLICATION_LOCK_PATH: &str = ".rustfs-table/warehouses/default/publication.lock";
|
||||
@@ -97,6 +97,14 @@ Current guidance:
|
||||
- enables minimal payload mode for GET health responses (`status`, `ready` only).
|
||||
- `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
|
||||
- TTL for readiness cache evaluation.
|
||||
- `RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE`
|
||||
- withdraws readiness when bounded object read/write stages stop completing while requests remain active.
|
||||
- default is `true`.
|
||||
- `RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS`
|
||||
- maximum time without completion in a bounded object stage before readiness is withdrawn.
|
||||
- default is `30000`; `0` uses the default.
|
||||
- the effective value is at least 5 seconds longer than `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`.
|
||||
- this readiness SLO is independent of disk read/write failure deadlines and may withdraw traffic before those deadlines expire.
|
||||
- `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
|
||||
- enables busy protection behavior for health probes.
|
||||
- default is `false`.
|
||||
|
||||
@@ -353,6 +353,11 @@ pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
|
||||
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
|
||||
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
||||
|
||||
/// Default detailed PUT stage metrics enabled
|
||||
/// Default value: false
|
||||
/// Environment variable: RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED
|
||||
pub const DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED: bool = false;
|
||||
|
||||
/// Default logs export enabled
|
||||
/// It is used to enable or disable exporting logs
|
||||
/// Default value: true
|
||||
|
||||
@@ -22,6 +22,19 @@ pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
|
||||
pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
|
||||
pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
|
||||
|
||||
/// Enable readiness withdrawal when bounded object read/write stages stop
|
||||
/// completing while requests remain active.
|
||||
pub const ENV_HEALTH_OBJECT_PROGRESS_ENABLE: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE";
|
||||
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_ENABLE: bool = true;
|
||||
|
||||
/// Requested time without completion in a bounded object stage before local
|
||||
/// readiness is withdrawn (milliseconds). A value of `0` uses the default;
|
||||
/// runtime adds a safety floor based on the object-lock acquisition timeout.
|
||||
pub const ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS";
|
||||
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
|
||||
/// Additional time beyond the configured object-lock acquisition deadline.
|
||||
pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
|
||||
|
||||
/// Timeout for cluster health readiness collectors (milliseconds).
|
||||
/// This bounds expensive storage and lock quorum checks used by cluster probes.
|
||||
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";
|
||||
|
||||
@@ -137,6 +137,37 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
/// Request the object-transaction fencing contract used by storage-owned
|
||||
/// cleanup receipts and lock-window optimizations.
|
||||
///
|
||||
/// This is fail-closed: enabling the writer without a live fleet proof rejects
|
||||
/// the commit rather than silently using a legacy-safe path.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_WRITE: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// object transaction fencing contract.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE);
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED);
|
||||
|
||||
/// Request preserving legacy per-part checksum metadata during data movement.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// data-movement per-part checksum sidecar.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -649,4 +680,22 @@ mod remote_version_state_tests {
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED,
|
||||
"RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_transaction_fencing_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_OBJECT_TRANSACTION_FENCING_WRITE, "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED,
|
||||
"RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,6 +81,9 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
|
||||
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
||||
/// Runtime env var controlling the transition worker count.
|
||||
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
||||
/// Runtime env var controlling the ILM expiry worker count. A set, parsable,
|
||||
/// non-zero value wins; anything else falls back to `min(cpus, 16)`.
|
||||
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
|
||||
/// Runtime env var controlling the absolute maximum transition workers.
|
||||
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
||||
/// Runtime env var controlling the transition queue capacity.
|
||||
|
||||
@@ -36,6 +36,11 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
|
||||
/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
|
||||
pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;
|
||||
|
||||
/// Environment variable for an extra outbound root CA certificate bundle.
|
||||
/// Use this to trust an internal CA for outbound HTTPS clients without replacing
|
||||
/// the default operating-system/web PKI roots via SSL_CERT_FILE.
|
||||
pub const ENV_RUSTFS_EXTRA_CA_CERT: &str = "RUSTFS_EXTRA_CA_CERT";
|
||||
|
||||
/// Environment variable to trust leaf certificates as CA
|
||||
/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation.
|
||||
/// By default, this is disabled.
|
||||
|
||||
@@ -44,6 +44,10 @@ pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENAB
|
||||
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
||||
|
||||
/// Enables detailed per-stage PUT metrics. Disabled by default because each
|
||||
/// PUT records multiple timers and histograms when attribution is active.
|
||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
@@ -141,6 +145,7 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -37,7 +37,6 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
||||
hotpath.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -846,8 +846,15 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Data usage cache info
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
/// Read-only projection of the scanner's `.usage-cache.bin` info block.
|
||||
///
|
||||
/// The canonical wire format is written by the hand-written map-encoded
|
||||
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
||||
/// This type decodes only the shared subset and is deliberately not
|
||||
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
||||
/// corrupt the cache for scanner readers, so no write path may exist here.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
pub name: String,
|
||||
pub next_cycle: u64,
|
||||
@@ -863,8 +870,12 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Data usage cache
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
/// owns the persisted format; this type only decodes it (see
|
||||
/// [`DataUsageCacheInfo`]) and must never grow a serialization path.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCache {
|
||||
pub info: DataUsageCacheInfo,
|
||||
pub cache: HashMap<String, DataUsageEntry>,
|
||||
@@ -1186,31 +1197,10 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let t: Self = rmp_serde::from_slice(buf)?;
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
// Note: load and save methods are storage-specific and should be implemented
|
||||
// in the ecstore crate where storage access is available
|
||||
}
|
||||
|
||||
/// Trait for storage-specific operations on DataUsageCache
|
||||
#[async_trait::async_trait]
|
||||
pub trait DataUsageCacheStorage {
|
||||
/// Load data usage cache from backend storage
|
||||
async fn load(store: &dyn std::any::Any, name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
Self: Sized;
|
||||
|
||||
/// Save data usage cache to backend storage
|
||||
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
}
|
||||
|
||||
// Helper structs and functions for cache operations
|
||||
@@ -1832,6 +1822,82 @@ mod tests {
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
/// Scanner-written `.usage-cache.bin` bytes: a 2-element array of the
|
||||
/// canonical 16-field map-encoded info block and one map-encoded entry.
|
||||
/// Captured from the canonical writer's `marshal_msg` — see
|
||||
/// `usage_cache_wire_format_is_pinned` in
|
||||
/// `crates/scanner/src/data_usage_define.rs`, which pins these exact
|
||||
/// bytes and documents regeneration. Hardcoded here because a
|
||||
/// dev-dependency on rustfs-scanner would pull the whole ecstore tree
|
||||
/// into this crate's test build, and a fixture generated at test runtime
|
||||
/// could not detect writer drift anyway.
|
||||
const SCANNER_USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn thin_usage_cache_decodes_scanner_wire_fixture() {
|
||||
let decoded =
|
||||
DataUsageCache::unmarshal(SCANNER_USAGE_CACHE_WIRE_FIXTURE).expect("thin projection decodes a scanner-written cache");
|
||||
|
||||
// The six fields shared with the scanner's 16-field info block; the
|
||||
// remaining ten (lifecycle, replication, checkpoint, heals, ...) must
|
||||
// be skipped, not error.
|
||||
assert_eq!(decoded.info.name, "wire-bucket");
|
||||
assert_eq!(decoded.info.next_cycle, 7);
|
||||
assert_eq!(
|
||||
decoded.info.last_update,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||
);
|
||||
assert!(decoded.info.skip_healing);
|
||||
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||
assert!(decoded.info.snapshot_complete);
|
||||
|
||||
// Entries use the shared canonical map-encoded type end to end.
|
||||
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||
assert_eq!(entry.size, 4096);
|
||||
assert_eq!(entry.objects, 3);
|
||||
assert_eq!(entry.versions, 5);
|
||||
assert_eq!(entry.delete_markers, 1);
|
||||
assert!(entry.compacted);
|
||||
assert_eq!(entry.failed_objects, 2);
|
||||
assert_eq!(
|
||||
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||
Some(&TierStats {
|
||||
total_size: 2048,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
+102
-18
@@ -40,7 +40,8 @@ use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::BTreeSet;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
@@ -59,13 +60,26 @@ pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
impl VersionShardCensus {
|
||||
pub(crate) fn is_complete(&self) -> bool {
|
||||
self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers
|
||||
self.has_xl_meta
|
||||
&& self.expected_part_numbers.len() == self.present_part_fingerprints.len()
|
||||
&& self
|
||||
.expected_part_numbers
|
||||
.iter()
|
||||
.all(|part_number| self.present_part_fingerprints.contains_key(part_number))
|
||||
}
|
||||
|
||||
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
||||
@@ -73,10 +87,25 @@ impl VersionShardCensus {
|
||||
&& self.is_complete()
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
fn shard_fingerprint(data: &[u8]) -> ChaosResult<PartShardFingerprint> {
|
||||
Ok(PartShardFingerprint {
|
||||
size: u64::try_from(data.len())?,
|
||||
sha256: sha256_hex(data),
|
||||
})
|
||||
}
|
||||
|
||||
/// Single-node RustFS server with `disk_count` local volume directories that
|
||||
/// can be faulted individually while the server is running.
|
||||
pub struct DiskFaultHarness {
|
||||
@@ -283,8 +312,10 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -296,20 +327,31 @@ pub(crate) fn census_object_version_on_disk(
|
||||
file_info.parts.iter().map(|part| part.number).collect()
|
||||
};
|
||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||
let erasure_index = Some(file_info.erasure.index);
|
||||
let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
|
||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||
let present_part_numbers = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
entry
|
||||
.file_type()
|
||||
.ok()
|
||||
.filter(|kind| kind.is_file())
|
||||
.and_then(|_| entry.file_name().to_str().map(str::to_owned))
|
||||
})
|
||||
.filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::<usize>().ok()))
|
||||
.collect(),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(),
|
||||
let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => {
|
||||
let mut fingerprints = BTreeMap::new();
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
if !entry.file_type()?.is_file() {
|
||||
continue;
|
||||
}
|
||||
let file_name = entry.file_name();
|
||||
let Some(part_number) = file_name
|
||||
.to_str()
|
||||
.and_then(|name| name.strip_prefix("part."))
|
||||
.and_then(|number| number.parse::<usize>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let data = std::fs::read(entry.path())?;
|
||||
fingerprints.insert(part_number, shard_fingerprint(&data)?);
|
||||
}
|
||||
fingerprints
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
|
||||
@@ -317,8 +359,10 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
expected_part_numbers,
|
||||
present_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -358,3 +402,43 @@ pub async fn signed_admin_post(url: &str, body: Option<&str>, access_key: &str,
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn complete_census() -> VersionShardCensus {
|
||||
VersionShardCensus {
|
||||
version_id: Some("version".to_string()),
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shard_fingerprint_uses_physical_length_and_sha256() {
|
||||
assert_eq!(
|
||||
shard_fingerprint(b"abc").unwrap(),
|
||||
PartShardFingerprint {
|
||||
size: 3,
|
||||
sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_requires_matching_inline_payload() {
|
||||
let mut expected = complete_census();
|
||||
expected.expected_part_numbers.clear();
|
||||
expected.present_part_fingerprints.clear();
|
||||
expected.inline_data_fingerprint = Some(shard_fingerprint(b"expected").unwrap());
|
||||
let mut changed = expected.clone();
|
||||
changed.inline_data_fingerprint = Some(shard_fingerprint(b"changed").unwrap());
|
||||
assert!(expected.matches_manifest(&expected));
|
||||
assert!(!changed.matches_manifest(&expected));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,19 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(log_path) = log_path else {
|
||||
return Ok(());
|
||||
};
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_s3_config(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
@@ -557,13 +570,7 @@ impl RustFSTestEnvironment {
|
||||
for (key, value) in extra_env {
|
||||
command.env(key, value);
|
||||
}
|
||||
// Optionally capture the child's stdout+stderr to a file so the test can
|
||||
// grep server logs (e.g. to confirm which GET reader path was taken).
|
||||
if let Some(log_path) = &self.capture_log_path {
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
}
|
||||
capture_command_logs(&mut command, self.capture_log_path.as_deref())?;
|
||||
let process = command.args(&args).spawn()?;
|
||||
|
||||
self.process = Some(process);
|
||||
@@ -1051,6 +1058,7 @@ pub struct RustFSTestClusterEnvironment {
|
||||
pub secret_key: String,
|
||||
pub extra_env: Vec<(String, String)>,
|
||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||
pub node_capture_log_paths: Vec<Option<String>>,
|
||||
pub topology: ClusterTopology,
|
||||
}
|
||||
|
||||
@@ -1150,6 +1158,7 @@ impl RustFSTestClusterEnvironment {
|
||||
secret_key: "rustfs-cluster-test-secret".to_string(),
|
||||
extra_env,
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
})
|
||||
}
|
||||
@@ -1179,6 +1188,20 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Capture stdout+stderr for a single cluster node process.
|
||||
pub fn set_node_capture_log_path<P>(
|
||||
&mut self,
|
||||
node_idx: usize,
|
||||
path: P,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
P: Into<String>,
|
||||
{
|
||||
self.ensure_node_index(node_idx)?;
|
||||
self.node_capture_log_paths[node_idx] = Some(path.into());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if node_idx >= self.nodes.len() {
|
||||
return Err(format!("node_idx {node_idx} is invalid").into());
|
||||
@@ -1268,6 +1291,7 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[i] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
|
||||
@@ -1294,6 +1318,7 @@ impl RustFSTestClusterEnvironment {
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let volumes_arg = self.build_volumes_arg();
|
||||
let log_path = self.node_capture_log_paths[node_idx].clone();
|
||||
let node = &mut self.nodes[node_idx];
|
||||
info!("Starting cluster node {} on {}", node_idx, node.address);
|
||||
|
||||
@@ -1312,6 +1337,7 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[node_idx] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, log_path.as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
node.process = Some(process);
|
||||
@@ -1563,6 +1589,7 @@ mod tests {
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
extra_env: Vec::new(),
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
}
|
||||
}
|
||||
@@ -1658,6 +1685,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_log_capture_supports_per_node_paths() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(3));
|
||||
env.set_node_capture_log_path(1, "/tmp/node1.log").unwrap();
|
||||
assert_eq!(env.node_capture_log_paths[0], None);
|
||||
assert_eq!(env.node_capture_log_paths[1], Some("/tmp/node1.log".to_string()));
|
||||
assert_eq!(env.node_capture_log_paths[2], None);
|
||||
assert!(env.set_node_capture_log_path(3, "/tmp/invalid.log").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_env_rejects_invalid_index() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
@@ -25,6 +26,15 @@ fn generate_compressible_data(size: usize) -> Vec<u8> {
|
||||
data
|
||||
}
|
||||
|
||||
/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part
|
||||
/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload
|
||||
/// Pending truncation (rustfs/rustfs#5957).
|
||||
fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
|
||||
(0..size)
|
||||
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
|
||||
let bucket_path = PathBuf::from(temp_dir).join(bucket);
|
||||
let mut part_files = Vec::new();
|
||||
@@ -55,9 +65,14 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let process = Command::new(&binary_path)
|
||||
// Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR
|
||||
// capture (survives the temp-dir cleanup on Drop and is uploaded as a CI
|
||||
// artifact); without the env var the child inherits stdio as before.
|
||||
let mut command = Command::new(&binary_path);
|
||||
command
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
@@ -66,8 +81,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
])
|
||||
.spawn()?;
|
||||
]);
|
||||
crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?;
|
||||
let process = command.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
@@ -154,3 +170,647 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MULTIPART_COMPRESSION_BUCKET: &str = "compression-multipart-bucket";
|
||||
const MPU_PART1_SIZE: usize = 5 * 1024 * 1024;
|
||||
const MPU_PART2_SIZE: usize = 1024 * 1024;
|
||||
|
||||
async fn multipart_upload(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
parts: &[&[u8]],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::with_capacity(parts.len());
|
||||
for (i, part) in parts.iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.to_vec()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_range(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
range: &str,
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let response = client.get_object().bucket(bucket).key(key).range(range).send().await?;
|
||||
Ok(response.body.collect().await?.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// Multipart disk compression roundtrip: parts are written as independent
|
||||
/// compressed streams and every GET shape must reassemble the original bytes
|
||||
/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression
|
||||
/// entirely).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)");
|
||||
|
||||
// Full GET must reassemble both independently compressed parts.
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range fully inside part 1.
|
||||
let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?;
|
||||
assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// Range fully inside part 2.
|
||||
let part2_start = MPU_PART1_SIZE + 4096;
|
||||
let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1;
|
||||
let range_inside_part2 = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={part2_start}-{part2_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_inside_part2[..],
|
||||
&original_data[part2_start..part2_end + 1],
|
||||
"part-2 range mismatch"
|
||||
);
|
||||
|
||||
// Suffix range (last 128 KiB, entirely in part 2).
|
||||
let suffix_len = 128 * 1024;
|
||||
let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
// partNumber GETs must return each original part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
info!("Multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket";
|
||||
|
||||
/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the
|
||||
/// disk-compression path runs and each part is stored as many compressed blocks — the shape that
|
||||
/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return
|
||||
/// the exact original bytes, and the stored size must show the data really was compressed.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart high-ratio binary compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-high-ratio.txt";
|
||||
let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7);
|
||||
let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
|
||||
// margin still proves the parts were stored compressed rather than raw or double-encoded.
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size as u64) / 2,
|
||||
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
|
||||
);
|
||||
info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)");
|
||||
|
||||
info!("step: full GET");
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
info!("step: boundary range GET");
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_HIGH_RATIO_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
info!("step: partNumber GET");
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch");
|
||||
|
||||
info!("Multipart high-ratio binary compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket";
|
||||
const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024;
|
||||
const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied
|
||||
/// range must be decompressed on read and re-compressed into the destination part, so the final
|
||||
/// object has to match "source prefix + uploaded tail" byte for byte.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart upload-part-copy compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
|
||||
// Source object: a plain PUT that goes through the single-stream compression path.
|
||||
let source_key = "copy-source.txt";
|
||||
let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(source_key)
|
||||
.body(ByteStream::from(source_data.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Destination object: part 1 copied from the source, part 2 uploaded directly.
|
||||
let target_key = "copy-target.txt";
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec();
|
||||
expected_data.extend_from_slice(&part2);
|
||||
let total_size = expected_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let copy_part = client
|
||||
.upload_part_copy()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}"))
|
||||
.copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1))
|
||||
.send()
|
||||
.await?;
|
||||
let copy_etag = copy_part
|
||||
.copy_part_result()
|
||||
.and_then(|r| r.e_tag())
|
||||
.ok_or("missing copy part etag")?
|
||||
.to_string();
|
||||
|
||||
let uploaded_part = client
|
||||
.upload_part()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build())
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_part.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch");
|
||||
|
||||
info!("Multipart upload-part-copy compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket";
|
||||
const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024;
|
||||
|
||||
/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one
|
||||
/// compressed part stream, and a suffix range must resolve inside the trailing part.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting three-part multipart compression partNumber test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-three-parts.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
original_data.extend_from_slice(&part3);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
|
||||
// Every partNumber GET must return exactly the bytes of the corresponding uploaded part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
// Suffix range (last 64 KiB) resolves inside the trailing part.
|
||||
let suffix_len = 64 * 1024;
|
||||
let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
info!("Three-part multipart compression partNumber test passed");
|
||||
env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
|
||||
|
||||
async fn start_rustfs_with_compression_and_sse(
|
||||
env: &mut RustFSTestEnvironment,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use base64::Engine;
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
// Server output goes to a file inside the per-test temp dir so a failing
|
||||
// run can be diagnosed from the child's logs.
|
||||
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
|
||||
let server_log_err = server_log.try_clone()?;
|
||||
let process = Command::new(&binary_path)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
|
||||
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
|
||||
.stdout(std::process::Stdio::from(server_log))
|
||||
.stderr(std::process::Stdio::from(server_log_err))
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
"--access-key",
|
||||
&env.access_key,
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
])
|
||||
.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
||||
/// shape must still return the original plaintext bytes. Physical size must shrink because the
|
||||
/// compression runs before encryption.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
|
||||
init_logging();
|
||||
info!("Starting SSE-S3 multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression_and_sse(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-sse-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::new();
|
||||
for (i, part) in [&part1, &part2].into_iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
assert_eq!(
|
||||
head_response.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::Aes256),
|
||||
"HEAD must report SSE-S3"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary must decrypt and decompress across parts.
|
||||
let boundary_start = MPU_PART1_SIZE - 64 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_SSE_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"SSE-S3 boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch");
|
||||
|
||||
info!("SSE-S3 multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -30,10 +30,10 @@ use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||
HeadObjectInput, HeadObjectOutput, PutObjectInput, PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -76,6 +76,18 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
];
|
||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
];
|
||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||
|
||||
@@ -91,6 +103,7 @@ pub enum Operation {
|
||||
GetObject,
|
||||
HeadObject,
|
||||
DeleteObject,
|
||||
ListObjectVersions,
|
||||
CreateMultipartUpload,
|
||||
UploadPart,
|
||||
CompleteMultipartUpload,
|
||||
@@ -109,12 +122,33 @@ pub enum FaultAction {
|
||||
/// already have buffered the rest of the current frame; the journal reports
|
||||
/// the threshold, and the backend never receives or stores the request.
|
||||
DisconnectAfterBytes(usize),
|
||||
/// Apply the request, then close the connection before returning its response.
|
||||
DisconnectAfterResponse,
|
||||
/// Drain a request body in fixed-size slices, sleeping after every slice.
|
||||
SlowDrain { chunk_bytes: usize, delay: Duration },
|
||||
/// Store the request normally but replace the response ETag.
|
||||
WrongEtag,
|
||||
}
|
||||
|
||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||
/// sender-side tests can assert what a real target would receive.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReplicationTimestampHeaders {
|
||||
pub tagging: Option<String>,
|
||||
pub retention: Option<String>,
|
||||
pub legalhold: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicationTimestampHeaders {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Credential-free request metadata retained for deterministic assertions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestRecord {
|
||||
@@ -128,6 +162,7 @@ pub struct RequestRecord {
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
@@ -141,6 +176,8 @@ struct ControlState {
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
assign_own_multipart_version_ids: bool,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
total_bytes: usize,
|
||||
@@ -383,6 +420,22 @@ impl FakeS3Target {
|
||||
.is_some_and(|version| !version.delete_marker)
|
||||
}
|
||||
|
||||
/// Make the target mint its own version ids instead of mirroring the
|
||||
/// forwarded source version id — models a generic S3 service.
|
||||
pub fn assign_own_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_version_ids = enabled;
|
||||
}
|
||||
|
||||
/// Mint own version ids for the multipart path only — models a target
|
||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
|
||||
pub fn active_multipart_upload_count(&self) -> usize {
|
||||
lock(&self.backend.store).uploads.len()
|
||||
}
|
||||
|
||||
/// Queue `times` copies of a fault for one operation.
|
||||
pub fn inject(&self, operation: Operation, action: FaultAction, times: usize) {
|
||||
if times == 0 {
|
||||
@@ -433,6 +486,25 @@ impl FakeS3Target {
|
||||
lock(&self.control).requests.drain(..).collect()
|
||||
}
|
||||
|
||||
/// Stored versions for one key as `(version_id, is_delete_marker)`, oldest
|
||||
/// first. Empty when the bucket or key does not exist. Lets purge tests
|
||||
/// assert on the target's actual state instead of inferring it from the
|
||||
/// request journal (a versioned DELETE is a silent no-op for missing ids).
|
||||
pub fn stored_versions(&self, bucket: &str, key: &str) -> Vec<(String, bool)> {
|
||||
let state = lock(&self.backend.store);
|
||||
state
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.and_then(|bucket_state| bucket_state.objects.get(key))
|
||||
.map(|versions| {
|
||||
versions
|
||||
.iter()
|
||||
.map(|version| (version.version_id.clone(), version.delete_marker))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub async fn shutdown(mut self) {
|
||||
let _ = self.shutdown.send(true);
|
||||
if let Some(task) = self.task.take() {
|
||||
@@ -496,7 +568,15 @@ impl S3Access for FaultAccess {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok());
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||
let fault = record_request(
|
||||
&self.control,
|
||||
operation,
|
||||
context.method().clone(),
|
||||
parsed,
|
||||
content_length,
|
||||
replication_timestamps,
|
||||
);
|
||||
if let Some(RequestFault {
|
||||
action: FaultAction::Status(status),
|
||||
..
|
||||
@@ -549,6 +629,7 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
content_length: Option<u64>,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
) -> Option<RequestFault> {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
@@ -573,6 +654,7 @@ fn record_request(
|
||||
part_number: parsed.part_number,
|
||||
content_length,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps,
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
@@ -633,6 +715,7 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
let operation = match (method, key.is_some()) {
|
||||
(&Method::HEAD, false) => Operation::HeadBucket,
|
||||
(&Method::GET, false) if query.contains_key("versioning") => Operation::GetBucketVersioning,
|
||||
(&Method::GET, false) if query.contains_key("versions") => Operation::ListObjectVersions,
|
||||
(&Method::PUT, true) if upload_id.is_some() && part_number.is_some() => Operation::UploadPart,
|
||||
(&Method::PUT, true) if upload_id.is_some() || query.contains_key("partNumber") => Operation::Unknown,
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
@@ -689,10 +772,17 @@ fn validate_retained_identifier(value: String, field: &str) -> S3Result<String>
|
||||
}
|
||||
}
|
||||
|
||||
fn new_version_id(headers: &HeaderMap) -> S3Result<String> {
|
||||
/// `assign_own` models a target that mints its own version ids (a generic S3
|
||||
/// service): the forwarded source-version-id header is validated but NOT
|
||||
/// mirrored into the stored version.
|
||||
fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
||||
let Some(value) = header_value(headers, &SOURCE_VERSION_ID_HEADERS) else {
|
||||
return Ok(Uuid::new_v4().to_string());
|
||||
};
|
||||
if assign_own {
|
||||
validate_retained_identifier(value.trim().to_owned(), "source version ID")?;
|
||||
return Ok(Uuid::new_v4().to_string());
|
||||
}
|
||||
let value = validate_retained_identifier(value.trim().to_owned(), "source version ID")?;
|
||||
let version_id = Uuid::parse_str(&value).map_err(|_| s3s::s3_error!(InvalidArgument, "source version ID must be a UUID"))?;
|
||||
Ok(version_id.to_string())
|
||||
@@ -777,7 +867,10 @@ async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<Cont
|
||||
update_consumed(control, fault.expect("matched fault").sequence, 0);
|
||||
Err(scripted_disconnect_error())
|
||||
}
|
||||
Some(FaultAction::SlowDrain { .. }) | Some(FaultAction::WrongEtag) | None => Ok(()),
|
||||
Some(FaultAction::SlowDrain { .. })
|
||||
| Some(FaultAction::WrongEtag)
|
||||
| Some(FaultAction::DisconnectAfterResponse)
|
||||
| None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -818,7 +911,7 @@ async fn collect_stream(
|
||||
Some(FaultAction::SlowDrain { chunk_bytes, delay }) => {
|
||||
return collect_stream_slow(body, capacity, *chunk_bytes, *delay).await;
|
||||
}
|
||||
Some(FaultAction::WrongEtag) | None => {}
|
||||
Some(FaultAction::WrongEtag) | Some(FaultAction::DisconnectAfterResponse) | None => {}
|
||||
}
|
||||
|
||||
let mut output = BytesMut::with_capacity(capacity);
|
||||
@@ -880,6 +973,9 @@ fn apply_response_fault<T>(mut response: S3Response<T>, fault: Option<&RequestFa
|
||||
if fault.is_some_and(|fault| fault.action == FaultAction::WrongEtag) {
|
||||
response.headers.insert(ETAG, HeaderValue::from_static(WRONG_ETAG));
|
||||
}
|
||||
if fault.is_some_and(|fault| fault.action == FaultAction::DisconnectAfterResponse) {
|
||||
response.headers.insert(DISCONNECT_HEADER, HeaderValue::from_static("true"));
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
@@ -1068,6 +1164,63 @@ impl S3 for FakeBackend {
|
||||
))
|
||||
}
|
||||
|
||||
/// Prefix + max-keys subset only — enough for the replication-check probe
|
||||
/// key allocation. No pagination markers or delimiter folding.
|
||||
async fn list_object_versions(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let state = lock(&self.store);
|
||||
let Some(bucket_state) = state.buckets.get(&req.input.bucket) else {
|
||||
return Err(s3s::s3_error!(NoSuchBucket, "bucket does not exist"));
|
||||
};
|
||||
let prefix = req.input.prefix.as_deref().unwrap_or_default();
|
||||
let max_keys = req.input.max_keys.unwrap_or(1000).max(0) as usize;
|
||||
|
||||
let mut keys: Vec<&String> = bucket_state.objects.keys().filter(|key| key.starts_with(prefix)).collect();
|
||||
keys.sort();
|
||||
|
||||
let mut versions = Vec::new();
|
||||
let mut delete_markers = Vec::new();
|
||||
'keys: for key in keys {
|
||||
for version in bucket_state.objects[key].iter().rev() {
|
||||
if versions.len() + delete_markers.len() >= max_keys {
|
||||
break 'keys;
|
||||
}
|
||||
if version.delete_marker {
|
||||
delete_markers.push(DeleteMarkerEntry {
|
||||
key: Some(key.clone()),
|
||||
version_id: Some(ObjectVersionId::from(version.version_id.clone())),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
..Default::default()
|
||||
});
|
||||
} else {
|
||||
versions.push(s3s::dto::ObjectVersion {
|
||||
key: Some(key.clone()),
|
||||
version_id: Some(ObjectVersionId::from(version.version_id.clone())),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
e_tag: Some(ETag::Strong(version.e_tag.clone())),
|
||||
size: Some(version.body.len() as i64),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(state);
|
||||
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(ListObjectVersionsOutput {
|
||||
name: Some(req.input.bucket),
|
||||
versions: Some(versions),
|
||||
delete_markers: Some(delete_markers),
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
let _body_permit = timeout(MAX_FAULT_DURATION, Arc::clone(&self.body_limit).acquire_owned())
|
||||
@@ -1078,7 +1231,8 @@ impl S3 for FakeBackend {
|
||||
let input = req.input;
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let version_id = new_version_id(&headers)?;
|
||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||
let version_id = new_version_id(&headers, assign_own)?;
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
@@ -1121,7 +1275,7 @@ impl S3 for FakeBackend {
|
||||
content_type: version.content_type,
|
||||
metadata: version.metadata,
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1143,7 +1297,7 @@ impl S3 for FakeBackend {
|
||||
content_type: version.content_type,
|
||||
metadata: version.metadata,
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1212,7 +1366,9 @@ impl S3 for FakeBackend {
|
||||
));
|
||||
}
|
||||
|
||||
let version_id = new_version_id(&headers)?;
|
||||
// `state` is the live store guard: read the flag from it. Re-locking
|
||||
// would self-deadlock (the store mutex is not reentrant).
|
||||
let version_id = new_version_id(&headers, state.assign_own_version_ids)?;
|
||||
upsert_version(
|
||||
&mut state,
|
||||
&input.bucket,
|
||||
@@ -1252,12 +1408,16 @@ impl S3 for FakeBackend {
|
||||
ensure_upload_budget(&state)?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let upload_id = Uuid::new_v4().to_string();
|
||||
// Read the flag before the mutable borrow of `state.uploads` below
|
||||
// (and never re-lock the store: the mutex is not reentrant).
|
||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||
let version_id = new_version_id(&headers, mint_own)?;
|
||||
state.uploads.insert(
|
||||
upload_id.clone(),
|
||||
MultipartState {
|
||||
bucket: input.bucket.clone(),
|
||||
key: input.key.clone(),
|
||||
version_id: new_version_id(&headers)?,
|
||||
version_id,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
parts: BTreeMap::new(),
|
||||
@@ -1581,6 +1741,52 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("plain")
|
||||
.body(ByteStream::from_static(b"plain"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("stamped")
|
||||
.body(ByteStream::from_static(b"stamped"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let requests = target.requests();
|
||||
let plain = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||
.expect("plain PUT must be journaled");
|
||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||
|
||||
let stamped = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||
.expect("stamped PUT must be journaled");
|
||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! assert_sdk_error {
|
||||
($error:expr, $status:expr, $code:expr) => {{
|
||||
let error = &$error;
|
||||
@@ -2867,6 +3073,7 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
Some(0),
|
||||
ReplicationTimestampHeaders::default(),
|
||||
);
|
||||
}
|
||||
let records = lock(&control).requests.clone();
|
||||
@@ -2888,6 +3095,7 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
None,
|
||||
ReplicationTimestampHeaders::default(),
|
||||
);
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
|
||||
@@ -189,8 +189,6 @@ mod tests {
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
|
||||
// Lower the min-size floor so every non-inline object below is eligible.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
|
||||
// Route multipart objects through per-part codec streaming too.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
|
||||
// Lock optimization is on by default, but pin it so the gate's
|
||||
@@ -315,6 +313,13 @@ mod tests {
|
||||
},
|
||||
payload(64 * 1024, 2),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "small-non-inline-256kib-plus",
|
||||
expect_large: true,
|
||||
},
|
||||
payload(256 * 1024 + 1, 6),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "mid-1_5mib",
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use serial_test::serial;
|
||||
@@ -32,6 +32,56 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let invalid_groups = [
|
||||
("test group", "group name contains whitespace"),
|
||||
("test=group", "group name contains reserved characters =,"),
|
||||
("test,group", "group name contains reserved characters =,"),
|
||||
];
|
||||
|
||||
for (group, expected_message) in invalid_groups {
|
||||
let body = serde_json::json!({
|
||||
"group": group,
|
||||
"members": [],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
let (status, response_body) = admin_request(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(body),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"invalid group {group:?} must return HTTP 400, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains("<Code>InvalidArgument</Code>"),
|
||||
"invalid group {group:?} must return InvalidArgument, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains(&format!("<Message>{expected_message}</Message>")),
|
||||
"invalid group {group:?} returned an unexpected message: {response_body}"
|
||||
);
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
|
||||
@@ -1828,33 +1828,36 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
||||
/// Reverting the multipart compression fix must fail this test.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
|
||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-fallback";
|
||||
let bucket = "inline-multipart-compression-roundtrip";
|
||||
cluster.create_test_bucket(bucket).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
let key = "multipart/compression-disabled.txt";
|
||||
let key = "multipart/compressed.txt";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1871,6 +1874,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
||||
cluster.start().await?;
|
||||
|
||||
@@ -1890,14 +1894,21 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
ReaderPathExpectation::for_class(
|
||||
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
|
||||
LEGACY_DUPLEX,
|
||||
MULTIPART,
|
||||
COMPRESSED,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
PartNumberReaderPathExpectation::new(
|
||||
bucket,
|
||||
multipart_key,
|
||||
&second_part,
|
||||
multipart_body.len(),
|
||||
COMPRESSED,
|
||||
LEGACY_DUPLEX,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||
@@ -2353,7 +2364,11 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
hot_client.create_bucket().bucket(bucket).send().await?;
|
||||
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
|
||||
|
||||
let key = "transition/mixed-multipart.bin";
|
||||
// `.zip` sits on the disk-compression exclusion list: this test pins
|
||||
// msgpack compat controls across ILM transition, and a compressed object
|
||||
// would classify as `compressed` instead of `remote` (and the warm-tier
|
||||
// read path does not decode compression — tracked separately).
|
||||
let key = "transition/mixed-multipart.zip";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
|
||||
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
|
||||
assert!(
|
||||
|
||||
@@ -0,0 +1,611 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ILM on SSE-KMS buckets while per-key SSE authorization is enforced (backlog#1582).
|
||||
//!
|
||||
//! Per-key KMS authorization (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true`) scopes the
|
||||
//! SSE-KMS data path to the requesting principal's `kms:GenerateDataKey` /
|
||||
//! `kms:Decrypt` grants. Internal callers — the lifecycle scanner's expiry deletes
|
||||
//! and the tier transition worker's reads — carry no request principal, and
|
||||
//! `authorize_sse_kms_key` (rustfs/src/storage/sse.rs) exempts a `None` principal
|
||||
//! so background maintenance keeps working on encrypted buckets.
|
||||
//!
|
||||
//! These tests pin that exemption end to end. If enforcement ever starts applying
|
||||
//! to the scanner's internal operations, expiry stops happening on SSE-KMS buckets
|
||||
//! and [`ilm_expiration_on_sse_kms_bucket_under_enforcement`] times out; if it
|
||||
//! starts applying to the transition worker or the read-through path,
|
||||
//! [`ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back`] fails at the
|
||||
//! transition wait or the plaintext round-trip.
|
||||
//!
|
||||
//! The replication half of the same acceptance item lives in
|
||||
//! `crates/e2e_test/src/replication_extension_test.rs`
|
||||
//! (`test_bucket_replication_sse_kms_failure_contract`); ILM had no coverage
|
||||
//! before this file.
|
||||
//!
|
||||
//! Deployment constraint pinned by the transition test's setup: the RustFS warm
|
||||
//! backend forwards the object's stored `x-amz-server-side-encryption*` metadata
|
||||
//! as raw headers on the tier data PUT (`build_transition_put_options` +
|
||||
//! `api_put_object.rs` header mapping), so a RustFS tier target must itself have
|
||||
//! KMS enabled and hold the named key or it rejects every transition upload with
|
||||
//! 400 InvalidRequest. That rejection is independent of the enforcement switch;
|
||||
//! the cold server here therefore runs its own Local KMS with the same key id.
|
||||
|
||||
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, RestoreRequest,
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Transition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serial_test::serial;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const SSE_KEY: &str = "kms-ilm-sse-key";
|
||||
const PAYLOAD: &[u8] = b"kms ilm sse payload: survives enforcement, expires and transitions on schedule";
|
||||
|
||||
const EXPIRY_BUCKET: &str = "kms-ilm-expiry";
|
||||
const EXPIRE_KEY: &str = "expire/object.bin";
|
||||
const SURVIVOR_KEY: &str = "keep/object.bin";
|
||||
|
||||
const TIER_NAME: &str = "KMSCOLD";
|
||||
const TIER_BUCKET: &str = "kms-ilm-cold-tier";
|
||||
const TIER_PREFIX: &str = "tiered";
|
||||
const TRANSITION_BUCKET: &str = "kms-ilm-transition";
|
||||
const TRANSITION_KEY: &str = "tier/object.bin";
|
||||
|
||||
/// Generous CI safety net; with a 1s scanner cycle and 2s lifecycle days the
|
||||
/// terminal state normally lands within a few seconds.
|
||||
const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
|
||||
|
||||
/// Start a Local-KMS server with per-key SSE authorization enforced and the
|
||||
/// lifecycle clock accelerated.
|
||||
///
|
||||
/// KMS wiring matches `kms_authorization_negative_matrix_test.rs` (local backend,
|
||||
/// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
|
||||
/// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
|
||||
/// so a `Days=1` rule is due about two seconds after the write.
|
||||
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
|
||||
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
|
||||
|
||||
let key_dir = env.kms_keys_dir.clone();
|
||||
let args = vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
key_dir.as_str(),
|
||||
"--kms-default-key-id",
|
||||
SSE_KEY,
|
||||
];
|
||||
|
||||
let envs = [
|
||||
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
|
||||
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_ILM_PROCESS_TIME", "1"),
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
|
||||
];
|
||||
|
||||
env.base_env.start_rustfs_server_with_env(args, &envs).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the bucket's default encryption to SSE-KMS under [`SSE_KEY`], so plain
|
||||
/// PUTs (and internal rewrites) are encrypted without per-request SSE headers.
|
||||
async fn set_bucket_default_sse_kms(client: &Client, bucket: &str) -> TestResult {
|
||||
let encryption_config = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::AwsKms)
|
||||
.kms_master_key_id(SSE_KEY)
|
||||
.build()?,
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(encryption_config)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Assert via `HeadObject` that the stored object is SSE-KMS encrypted under
|
||||
/// [`SSE_KEY`]. Without this, a bucket-default misconfiguration would let the
|
||||
/// tests pass on an unencrypted object and prove nothing about KMS.
|
||||
async fn assert_head_sse_kms(client: &Client, bucket: &str, key: &str) -> TestResult {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(
|
||||
head.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"{bucket}/{key} must be SSE-KMS encrypted via the bucket default"
|
||||
);
|
||||
assert_eq!(
|
||||
head.ssekms_key_id(),
|
||||
Some(SSE_KEY),
|
||||
"{bucket}/{key} must be wrapped under the configured KMS key"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns `true` once `GET bucket/key` fails with `NoSuchKey`, `false` while it
|
||||
/// still succeeds. Any other error is surfaced. (Copied from
|
||||
/// `reliant/lifecycle.rs`; that helper is private to the reliant module.)
|
||||
async fn object_is_gone(client: &Client, bucket: &str, key: &str) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
|
||||
match client.get_object().bucket(bucket).key(key).send().await {
|
||||
Ok(output) => {
|
||||
output.body.collect().await?;
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(service_error) = e.as_service_error() {
|
||||
if service_error.is_no_such_key() {
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(format!("expected NoSuchKey, got: {e:?}").into());
|
||||
}
|
||||
Err(format!("expected a service error, got: {e:?}").into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll until `GET bucket/key` returns `NoSuchKey`, or fail after `deadline`.
|
||||
async fn wait_for_object_expired(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
if object_is_gone(client, bucket, key).await? {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} was not expired by the lifecycle scanner within {}s; \
|
||||
SSE key-policy enforcement may have started blocking the scanner's internal deletes",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a prefix-scoped `Days`-based expiration rule.
|
||||
async fn put_expiration_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.expiration(LifecycleExpiration::builder().days(days).build())
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Install a prefix-scoped `Days`-based transition rule targeting [`TIER_NAME`].
|
||||
async fn put_transition_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(days)
|
||||
.storage_class(TransitionStorageClass::from(TIER_NAME))
|
||||
.build(),
|
||||
)
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a plain Local-KMS server (no enforcement, no lifecycle acceleration)
|
||||
/// holding [`SSE_KEY`], to serve as the cold tier target.
|
||||
///
|
||||
/// The RustFS warm backend forwards the stored SSE-KMS headers on the tier data
|
||||
/// PUT, so the target re-applies managed SSE-KMS under the named key and must
|
||||
/// be able to resolve it; without KMS it answers 400 InvalidRequest and the
|
||||
/// transition can never complete. Enforcement stays off here: the tier writes
|
||||
/// arrive under `cold`'s root credentials, and one enforcing side is enough to
|
||||
/// pin the exemption.
|
||||
async fn start_cold_tier_kms_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
|
||||
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
|
||||
|
||||
let key_dir = env.kms_keys_dir.clone();
|
||||
let args = vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
key_dir.as_str(),
|
||||
"--kms-default-key-id",
|
||||
SSE_KEY,
|
||||
];
|
||||
|
||||
env.base_env
|
||||
.start_rustfs_server_with_env(args, &[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The subset of the manual transition run report these tests assert on.
|
||||
///
|
||||
/// Unknown fields are ignored, so this stays compatible with report growth; the
|
||||
/// full shape is pinned by `reliant/tiering.rs`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManualTransitionRunReport {
|
||||
#[serde(default)]
|
||||
scanned: u64,
|
||||
#[serde(default)]
|
||||
enqueued: u64,
|
||||
#[serde(default)]
|
||||
skipped_already_in_flight: u64,
|
||||
#[serde(default)]
|
||||
skipped_tier: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManualTransitionRunResponse {
|
||||
state: String,
|
||||
report: ManualTransitionRunReport,
|
||||
}
|
||||
|
||||
/// One synchronous (enqueue-only) manual transition run over `bucket/prefix`,
|
||||
/// via the same admin endpoint `reliant/tiering.rs` drives.
|
||||
async fn manual_transition_run(
|
||||
hot: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> Result<ManualTransitionRunResponse, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let bucket = urlencoding::encode(bucket);
|
||||
let prefix = urlencoding::encode(prefix);
|
||||
let tier = urlencoding::encode(TIER_NAME);
|
||||
let path =
|
||||
format!("/rustfs/admin/v3/ilm/transition/run?bucket={bucket}&prefix={prefix}&tier={tier}&dryRun=false&maxObjects=10");
|
||||
let (status, body) = admin_request(&hot.url, http::Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("manual transition run failed: status={status}, body={body}").into());
|
||||
}
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
/// Drive manual transition runs until one reports the object as processed.
|
||||
///
|
||||
/// The `Days=1` rule becomes due about two seconds after the write
|
||||
/// (`RUSTFS_ILM_DEBUG_DAY_SECS=2`), so early runs may legitimately report the
|
||||
/// object as not yet eligible; the loop keeps running the endpoint until it
|
||||
/// either enqueues the transition, sees it already in flight (the 1s scanner
|
||||
/// backstop got there first), or finds it already on the tier.
|
||||
async fn run_manual_transition_until_processed(
|
||||
hot: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
deadline: StdDuration,
|
||||
) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let run = manual_transition_run(hot, bucket, prefix).await?;
|
||||
assert_eq!(run.report.scanned, 1, "manual transition run must scan the object: {run:#?}");
|
||||
if run.report.enqueued + run.report.skipped_already_in_flight + run.report.skipped_tier >= 1 {
|
||||
info!(state = %run.state, report = ?run.report, "manual transition run processed the SSE-KMS object");
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"manual transition runs never processed {bucket}/{prefix} within {}s; last report: {run:#?}",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
|
||||
///
|
||||
/// No `force`, so the server runs the real connectivity probe against `cold`
|
||||
/// (the tier bucket must already exist there). Mirrors
|
||||
/// `reliant/tiering.rs::add_rustfs_tier`, which is private to that module.
|
||||
async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
|
||||
let body = serde_json::json!({
|
||||
"type": "rustfs",
|
||||
"rustfs": {
|
||||
"name": TIER_NAME,
|
||||
"endpoint": cold.url.as_str(),
|
||||
"accessKey": cold.access_key.as_str(),
|
||||
"secretKey": cold.secret_key.as_str(),
|
||||
"bucket": TIER_BUCKET,
|
||||
"prefix": TIER_PREFIX,
|
||||
"region": "us-east-1",
|
||||
"storageClass": ""
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let (status, resp) = admin_request(
|
||||
&hot.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/tier",
|
||||
Some(body),
|
||||
&hot.access_key,
|
||||
&hot.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Poll `HEAD` until the object's storage class is the tier name (transition
|
||||
/// complete), or fail after `deadline`. (From `reliant/tiering.rs`.)
|
||||
async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
if head.storage_class().map(|sc| sc.as_str()) == Some(TIER_NAME) {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} was not transitioned to {TIER_NAME} within {}s (storage_class={:?}); \
|
||||
SSE key-policy enforcement may have started blocking the transition worker's internal reads",
|
||||
deadline.as_secs(),
|
||||
head.storage_class()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll `HEAD` until `x-amz-restore` reports a finished restore
|
||||
/// (`ongoing-request="false"`), or fail after `deadline`.
|
||||
async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
if head.restore().is_some_and(|r| r.contains("ongoing-request=\"false\"")) {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} restore did not complete within {}s (restore={:?}); \
|
||||
SSE key-policy enforcement may have started blocking the restore copy-back's internal reads",
|
||||
deadline.as_secs(),
|
||||
head.restore()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// ILM expiration keeps working on an SSE-KMS bucket while per-key SSE
|
||||
/// authorization is enforced.
|
||||
///
|
||||
/// The lifecycle scanner deletes expired objects with an internal (no-principal)
|
||||
/// identity that holds no `kms` grant. If enforcement ever starts applying to
|
||||
/// those internal deletes (or to the scanner's metadata reads) on encrypted
|
||||
/// buckets, expiry stops happening and this test times out.
|
||||
///
|
||||
/// A survivor object under a non-matching prefix isolates the rule's prefix
|
||||
/// filter as the cause of the deletion and proves the encrypted bucket stays
|
||||
/// readable end to end after the scanner has run.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
|
||||
|
||||
let client = env.base_env.create_s3_client();
|
||||
set_bucket_default_sse_kms(&client, EXPIRY_BUCKET).await?;
|
||||
|
||||
for key in [EXPIRE_KEY, SURVIVOR_KEY] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(EXPIRY_BUCKET)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(PAYLOAD))
|
||||
.send()
|
||||
.await?;
|
||||
assert_head_sse_kms(&client, EXPIRY_BUCKET, key).await?;
|
||||
}
|
||||
info!("both objects stored SSE-KMS encrypted under enforcement");
|
||||
|
||||
put_expiration_rule(&client, EXPIRY_BUCKET, "kms-ilm-expire", "expire/", 1).await?;
|
||||
|
||||
// The regression this pins: the scanner's internal delete must stay exempt
|
||||
// from per-key SSE authorization, so the encrypted object actually expires.
|
||||
wait_for_object_expired(&client, EXPIRY_BUCKET, EXPIRE_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object expired by the lifecycle scanner under enforcement");
|
||||
|
||||
// Negative control: same bucket, same encryption, non-matching prefix. It
|
||||
// must survive the scanner and still decrypt for the requesting principal.
|
||||
assert!(
|
||||
!object_is_gone(&client, EXPIRY_BUCKET, SURVIVOR_KEY).await?,
|
||||
"non-matching-prefix object must not be expired by a prefix-scoped rule"
|
||||
);
|
||||
let survivor = client.get_object().bucket(EXPIRY_BUCKET).key(SURVIVOR_KEY).send().await?;
|
||||
assert_eq!(
|
||||
survivor.body.collect().await?.into_bytes().as_ref(),
|
||||
PAYLOAD,
|
||||
"surviving SSE-KMS object must still decrypt after the scanner has run"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// ILM transition to a remote tier keeps working on an SSE-KMS bucket while
|
||||
/// per-key SSE authorization is enforced, and the transitioned object reads
|
||||
/// back as plaintext.
|
||||
///
|
||||
/// The transition worker moves the stored (encrypted) bytes to the cold tier
|
||||
/// with an internal (no-principal) identity; the read-through `GET` then
|
||||
/// decrypts the envelope for the requesting principal. If enforcement ever
|
||||
/// starts applying to the worker's internal reads, the transition wait times
|
||||
/// out; if the stored envelope is mishandled across the tier round trip, the
|
||||
/// plaintext comparison fails.
|
||||
///
|
||||
/// The transition is driven through the manual transition-run admin endpoint
|
||||
/// (the mechanism `reliant/tiering.rs` established), so the test does not
|
||||
/// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
// Cold-tier server: independent credentials, its own Local KMS holding the
|
||||
// same key id (see the module docs for why the tier target needs KMS).
|
||||
// Started first; each server's startup cleanup only matches its own unique
|
||||
// address and temp dir, so the two instances coexist.
|
||||
let mut cold = LocalKMSTestEnvironment::new().await?;
|
||||
cold.base_env.access_key = "kmscoldtieradmin".to_string();
|
||||
cold.base_env.secret_key = "kmscoldtiersecret".to_string();
|
||||
start_cold_tier_kms_server(&mut cold).await?;
|
||||
let cold_client = cold.base_env.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
// Hot server: Local KMS + enforcement + accelerated lifecycle clock.
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
let hot_client = env.base_env.create_s3_client();
|
||||
|
||||
add_rustfs_tier(&env.base_env, &cold.base_env).await?;
|
||||
|
||||
env.base_env.create_test_bucket(TRANSITION_BUCKET).await?;
|
||||
set_bucket_default_sse_kms(&hot_client, TRANSITION_BUCKET).await?;
|
||||
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.body(ByteStream::from_static(PAYLOAD))
|
||||
.send()
|
||||
.await?;
|
||||
assert_head_sse_kms(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY).await?;
|
||||
info!("object stored SSE-KMS encrypted under enforcement");
|
||||
|
||||
// Days=1 is due ~2s after the write with RUSTFS_ILM_DEBUG_DAY_SECS=2.
|
||||
put_transition_rule(&hot_client, TRANSITION_BUCKET, "kms-ilm-transition", "tier/", 1).await?;
|
||||
|
||||
// Drive the transition deterministically via the manual run endpoint, then
|
||||
// wait for HEAD to report the tier as the object's storage class.
|
||||
run_manual_transition_until_processed(&env.base_env, TRANSITION_BUCKET, "tier/", ILM_DEADLINE).await?;
|
||||
wait_for_transition(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object transitioned to the remote tier under enforcement");
|
||||
|
||||
let head = hot_client
|
||||
.head_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
head.restore().is_none(),
|
||||
"a freshly transitioned object must not advertise x-amz-restore, got {:?}",
|
||||
head.restore()
|
||||
);
|
||||
|
||||
// The remote copy exists on the cold tier. The payload the tier holds is the
|
||||
// hot server's stored ciphertext, wrapped once more under the cold server's
|
||||
// own managed SSE-KMS layer (the forwarded headers re-request encryption).
|
||||
let remote = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
|
||||
assert!(!remote.contents().is_empty(), "cold-tier bucket must hold the transitioned object's data");
|
||||
|
||||
// Read-through GET under enforcement must succeed (not AccessDenied) and
|
||||
// keep advertising SSE-KMS. Its BODY is deliberately not compared here:
|
||||
// the transitioned read path skips managed-SSE decryption — a product gap
|
||||
// unrelated to enforcement — so a direct GET streams the stored ciphertext
|
||||
// (`new_getobjectreader` in crates/ecstore/src/client/object_api_utils.rs
|
||||
// hardcodes `is_encrypted = false` and never applies the
|
||||
// `ReadTransform::Encrypted` wrapping the hot-read path builds in
|
||||
// crates/ecstore/src/object_api/readers.rs). Plaintext recovery is pinned
|
||||
// through restore semantics below; when the read-through gap is fixed, a
|
||||
// byte assertion can be added here too.
|
||||
let read_through = hot_client
|
||||
.get_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
read_through.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"transitioned object must still report SSE-KMS on read-through"
|
||||
);
|
||||
let read_through_body = read_through.body.collect().await?.into_bytes();
|
||||
assert_eq!(
|
||||
read_through_body.len(),
|
||||
PAYLOAD.len(),
|
||||
"read-through GET must stream the object's full logical size under enforcement"
|
||||
);
|
||||
|
||||
// RestoreObject copies the ciphertext back from the tier under the original
|
||||
// envelope metadata; the restored copy is then served by the normal
|
||||
// decrypting read path. The copy-back runs with an internal (no-principal)
|
||||
// identity, so this also pins the exemption on the restore path. Days=300
|
||||
// because RUSTFS_ILM_DEBUG_DAY_SECS=2 accelerates the restored copy's
|
||||
// expiry as well (300 accelerated days == 600s of validity).
|
||||
hot_client
|
||||
.restore_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.restore_request(RestoreRequest::builder().days(300).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_restore_complete(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object restored from the remote tier under enforcement");
|
||||
|
||||
// The KMS-relevant half: the restored envelope decrypts back to the exact
|
||||
// plaintext for the requesting principal.
|
||||
let restored = hot_client
|
||||
.get_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
restored.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"restored object must still report SSE-KMS"
|
||||
);
|
||||
let body = restored.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), PAYLOAD, "restored SSE-KMS object must round-trip byte-identical plaintext");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -59,3 +59,6 @@ mod configured_roundtrip_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod kms_authorization_negative_matrix_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod kms_ilm_sse_kms_test;
|
||||
|
||||
@@ -39,6 +39,10 @@ pub mod fault_proxy;
|
||||
#[cfg(test)]
|
||||
mod reliability_disk_fault_test;
|
||||
|
||||
// Privileged Linux-only 3x4 replacement rebuild proof for rustfs#5869/#1791.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod replacement_privileged_e2e_test;
|
||||
|
||||
// dist-13 (backlog#1150/#1155): e2e regression net proving a large-object
|
||||
// degraded EC read never returns a silently truncated body (rustfs#4594/#4560/#4585).
|
||||
#[cfg(test)]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2854,7 +2854,7 @@ pub(crate) mod cmptst_30 {
|
||||
result
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[ignore = "timing-sensitive backend-pressure latency probe; run explicitly with --ignored"]
|
||||
#[tokio::test]
|
||||
async fn regression() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -252,6 +252,7 @@ impl QuotaTestEnv {
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -963,9 +964,27 @@ mod integration_tests {
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(complete_result.is_err());
|
||||
let complete_error = complete_result.expect_err("multipart completion above quota must be rejected");
|
||||
assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest"));
|
||||
assert!(!env.object_exists("over_quota.txt").await?);
|
||||
|
||||
let staged_parts = env
|
||||
.client
|
||||
.list_parts()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(staged_parts.parts().len(), 2, "quota rejection must preserve the multipart upload");
|
||||
env.client
|
||||
.abort_multipart_upload()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -349,11 +349,32 @@ mod tests {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let first_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 40)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_inline_version = first_inline
|
||||
.version_id()
|
||||
.ok_or("first inline PUT did not return a version ID")?;
|
||||
let second_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_inline_version = second_inline
|
||||
.version_id()
|
||||
.ok_or("second inline PUT did not return a version ID")?;
|
||||
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 41)))
|
||||
.body(ByteStream::from(payload(128 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_version = first.version_id().ok_or("first PUT did not return a version ID")?;
|
||||
@@ -361,16 +382,36 @@ mod tests {
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 42)))
|
||||
.body(ByteStream::from(payload(3 * 1024 * 1024, 42)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_version = second.version_id().ok_or("second PUT did not return a version ID")?;
|
||||
let delete = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
let delete_version = delete.version_id().ok_or("delete marker did not return a version ID")?;
|
||||
|
||||
let first_inline_census = harness.census_object_version(0, bucket, "versions/inline.bin", Some(first_inline_version))?;
|
||||
let second_inline_census =
|
||||
harness.census_object_version(0, bucket, "versions/inline.bin", Some(second_inline_version))?;
|
||||
let first_census = harness.census_object_version(0, bucket, key, Some(first_version))?;
|
||||
let first_other_disk_census = harness.census_object_version(1, bucket, key, Some(first_version))?;
|
||||
let second_census = harness.census_object_version(0, bucket, key, Some(second_version))?;
|
||||
let delete_census = harness.census_object_version(0, bucket, key, Some(delete_version))?;
|
||||
assert!(
|
||||
first_inline_census.is_complete() && second_inline_census.is_complete(),
|
||||
"inline version physical census is incomplete: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.present_part_fingerprints.is_empty() && second_inline_census.present_part_fingerprints.is_empty(),
|
||||
"inline versions must not select external shard files: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.inline_data_fingerprint.is_some() && second_inline_census.inline_data_fingerprint.is_some(),
|
||||
"inline versions must fingerprint payload bytes stored in xl.meta"
|
||||
);
|
||||
assert_ne!(
|
||||
first_inline_census.inline_data_fingerprint, second_inline_census.inline_data_fingerprint,
|
||||
"same-size inline versions with different payloads must retain distinct xl.meta fingerprints"
|
||||
);
|
||||
assert!(
|
||||
first_census.is_complete(),
|
||||
"first version physical census is incomplete: {first_census:?}"
|
||||
@@ -379,6 +420,14 @@ mod tests {
|
||||
second_census.is_complete(),
|
||||
"second version physical census is incomplete: {second_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_other_disk_census.is_complete(),
|
||||
"first version physical census on the second disk is incomplete: {first_other_disk_census:?}"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.erasure_index, first_other_disk_census.erasure_index,
|
||||
"physical census must preserve each disk's erasure index"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.data_dir, second_census.data_dir,
|
||||
"distinct object versions must select distinct physical data directories"
|
||||
@@ -387,6 +436,24 @@ mod tests {
|
||||
first_census.expected_part_numbers, second_census.expected_part_numbers,
|
||||
"same single-part shape should expose the same part numbers"
|
||||
);
|
||||
let first_part = first_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("first version did not expose a physical part fingerprint")?;
|
||||
let second_part = second_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("second version did not expose a physical part fingerprint")?;
|
||||
assert_ne!(
|
||||
first_part.size, second_part.size,
|
||||
"different shard lengths must retain their physical sizes"
|
||||
);
|
||||
assert_ne!(
|
||||
first_part.sha256, second_part.sha256,
|
||||
"different shard contents must retain their physical hashes"
|
||||
);
|
||||
assert!(
|
||||
delete_census.is_complete(),
|
||||
"delete marker physical census is incomplete: {delete_census:?}"
|
||||
@@ -396,7 +463,7 @@ mod tests {
|
||||
"delete marker must not declare object shards: {delete_census:?}"
|
||||
);
|
||||
assert!(
|
||||
delete_census.present_part_numbers.is_empty(),
|
||||
delete_census.present_part_fingerprints.is_empty(),
|
||||
"delete marker must not select stale object shards: {delete_census:?}"
|
||||
);
|
||||
Ok(())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -32,6 +32,11 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
# RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
|
||||
# state in a stock build (backlog#1832).
|
||||
list-chaos = []
|
||||
rio-v2 = ["dep:rustfs-rio-v2"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
|
||||
@@ -69,6 +69,7 @@ fn build_non_inline_writers(config: &BenchConfig) -> Vec<Option<BitrotWriterWrap
|
||||
fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
let configs = vec![
|
||||
BenchConfig::new(4 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(16 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(64 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(128 * 1024, 4, 2, 128 * 1024),
|
||||
];
|
||||
@@ -112,7 +113,12 @@ fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
erasure
|
||||
.clone()
|
||||
.encode_single_block_non_inline(reader, &mut writers, config.data_shards)
|
||||
.encode_single_block_non_inline_with_size_hint(
|
||||
reader,
|
||||
&mut writers,
|
||||
config.data_shards,
|
||||
config.payload_size,
|
||||
)
|
||||
.await
|
||||
.expect("single block candidate benchmark");
|
||||
});
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
TargetClient,
|
||||
TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,9 +61,11 @@ pub mod bucket {
|
||||
delete_manual_transition_scope_admission_if_current, load_manual_transition_job_record,
|
||||
load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
|
||||
manual_transition_job_lease_expired, manual_transition_scope_admission_lease_expired,
|
||||
manual_transition_scope_key, persist_manual_transition_job_progress, renew_manual_transition_job_lease,
|
||||
request_manual_transition_job_cancel, save_manual_transition_job_record,
|
||||
save_manual_transition_job_record_if_current, save_manual_transition_scope_admission_if_absent,
|
||||
manual_transition_scope_key, persist_manual_transition_job_progress,
|
||||
persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease,
|
||||
renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel,
|
||||
save_manual_transition_job_record, save_manual_transition_job_record_if_current,
|
||||
save_manual_transition_scope_admission_if_absent, update_manual_transition_job_record,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -129,16 +131,19 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -180,17 +185,18 @@ pub mod bucket {
|
||||
mrf_backlog_observability_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
@@ -274,14 +280,16 @@ pub mod cluster {
|
||||
}
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled};
|
||||
pub use crate::io_support::compress::{
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod config {
|
||||
pub mod com {
|
||||
pub use crate::config::com::{
|
||||
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config,
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
|
||||
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
|
||||
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
|
||||
@@ -308,11 +316,13 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||
@@ -342,7 +352,7 @@ pub mod disk {
|
||||
}
|
||||
|
||||
pub mod error {
|
||||
pub use crate::disk::error::{BitrotErrorType, DiskError, Error, FileAccessDeniedWithContext, Result};
|
||||
pub use crate::disk::error::{DiskError, Error, FileAccessDeniedWithContext, Result};
|
||||
}
|
||||
|
||||
pub mod error_reduce {
|
||||
@@ -399,8 +409,11 @@ pub mod metrics {
|
||||
}
|
||||
|
||||
pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
@@ -409,10 +422,10 @@ pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
|
||||
ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
@@ -460,7 +473,8 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,9 @@ use rustfs_utils::http::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -80,7 +82,6 @@ use tracing::warn;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
@@ -1450,7 +1451,7 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
/// `modify_before_signing`: the parameter becomes part of the SigV4 canonical
|
||||
/// request.
|
||||
fn append_version_id_query(uri: &str, version_id: &str) -> String {
|
||||
pub fn append_version_id_query(uri: &str, version_id: &str) -> String {
|
||||
let separator = if uri.contains('?') { '&' } else { '?' };
|
||||
format!("{uri}{separator}versionId={}", urlencoding::encode(version_id))
|
||||
}
|
||||
@@ -1476,9 +1477,12 @@ impl Default for AdvancedPutOptions {
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
source_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
retention_timestamp: OffsetDateTime::now_utc(),
|
||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||
// timestamp header for it, otherwise a receiver would treat an
|
||||
// unset category as a modification made right now.
|
||||
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
@@ -1675,6 +1679,16 @@ impl PutObjectOptions {
|
||||
);
|
||||
}
|
||||
|
||||
for (suffix, timestamp) in [
|
||||
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||
] {
|
||||
if timestamp.unix_timestamp() != 0 {
|
||||
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
|
||||
if self.internal.replication_request {
|
||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
@@ -1861,6 +1875,9 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1868,7 +1885,7 @@ impl TargetClient {
|
||||
size: i64,
|
||||
body: ByteStream,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<(), S3ClientError> {
|
||||
) -> Result<Option<String>, S3ClientError> {
|
||||
let mut headers = opts.header();
|
||||
|
||||
let builder = self.client.put_object();
|
||||
@@ -1903,7 +1920,7 @@ impl TargetClient {
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
|
||||
Err(e) => match e {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
let err = service_err.into_err();
|
||||
@@ -2839,6 +2856,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||
// last-writer-wins on these headers (object-api-options.go parses them
|
||||
// as RFC3339); a replica without them loses every conflict resolution.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||
opts.internal.tagging_timestamp = tagging;
|
||||
opts.internal.retention_timestamp = retention;
|
||||
opts.internal.legalhold_timestamp = legalhold;
|
||||
|
||||
let header = opts.header();
|
||||
for (suffix, expected) in [
|
||||
("source-replication-tagging-timestamp", tagging),
|
||||
("source-replication-retention-timestamp", retention),
|
||||
("source-replication-legalhold-timestamp", legalhold),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||
"replication put requests must carry the {suffix} header"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||
// make the receiver treat an unset category as a fresh modification.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
|
||||
let header = opts.header();
|
||||
for suffix in [
|
||||
"source-replication-tagging-timestamp",
|
||||
"source-replication-retention-timestamp",
|
||||
"source-replication-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||
"unset {suffix} must not be sent to replication targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
@@ -64,10 +64,41 @@ impl BucketDurabilityConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Default durability tier seeded into a newly created bucket's metadata
|
||||
/// (rustfs/backlog#1811). `relaxed` aligns new buckets with MinIO's default
|
||||
/// posture: object data is still fdatasynced, while xl.meta and directory-entry
|
||||
/// fsyncs follow the relaxed durability gate.
|
||||
pub const ENV_NEW_BUCKET_DURABILITY_MODE: &str = "RUSTFS_NEW_BUCKET_DURABILITY_MODE";
|
||||
pub const DEFAULT_NEW_BUCKET_DURABILITY_MODE: &str = BUCKET_DURABILITY_MODE_RELAXED;
|
||||
|
||||
/// The `durability.json` bytes to seed into a freshly created bucket's metadata.
|
||||
/// Empty means "no override" (the bucket then follows the global
|
||||
/// `RUSTFS_DURABILITY_MODE`); otherwise the serialized chosen tier. Operators
|
||||
/// can set `inherit` to disable the new-bucket override. Invalid values also
|
||||
/// fail closed to inherit the global mode instead of seeding a surprising tier.
|
||||
pub fn new_bucket_durability_config_json() -> Vec<u8> {
|
||||
let raw = std::env::var(ENV_NEW_BUCKET_DURABILITY_MODE).unwrap_or_else(|_| DEFAULT_NEW_BUCKET_DURABILITY_MODE.to_string());
|
||||
let mode = raw.trim();
|
||||
if mode.eq_ignore_ascii_case("inherit") || mode.is_empty() || !BucketDurabilityConfig::is_valid_mode(mode) {
|
||||
return Vec::new();
|
||||
}
|
||||
serde_json::to_vec(&BucketDurabilityConfig::new(mode)).expect("BucketDurabilityConfig serialization cannot fail")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn new_bucket_seeded_mode() -> Option<String> {
|
||||
let json = new_bucket_durability_config_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_slice::<BucketDurabilityConfig>(&json)
|
||||
.expect("new-bucket durability config must serialize")
|
||||
.normalized_mode()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
@@ -99,4 +130,33 @@ mod tests {
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_seeds_relaxed_when_unset() {
|
||||
temp_env::with_var_unset(ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(BUCKET_DURABILITY_MODE_RELAXED));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_honors_explicit_tiers() {
|
||||
for mode in [
|
||||
BUCKET_DURABILITY_MODE_STRICT,
|
||||
BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_NONE,
|
||||
] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(mode));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_can_inherit_global_mode() {
|
||||
for mode in ["inherit", "", "bogus"] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode(), None);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,6 +86,21 @@ where
|
||||
com::save_config_with_opts(api, file, data, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_with_opts_quiet<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::save_config_with_opts_quiet(api, file, data, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -45,6 +45,104 @@ const MANUAL_TRANSITION_JOB_LEASE_SECONDS: i128 = 60;
|
||||
const MANUAL_TRANSITION_LEGACY_SCOPE_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_TASK_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_WORKER_RESULT_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_JOB_CAS_RETRIES: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
struct ManualTransitionJobCasBarrierState {
|
||||
job_id: Uuid,
|
||||
paused: std::sync::atomic::AtomicBool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Semaphore,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ManualTransitionJobCasBarrier {
|
||||
state: Arc<ManualTransitionJobCasBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static MANUAL_TRANSITION_JOB_CAS_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<ManualTransitionJobCasBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl ManualTransitionJobCasBarrier {
|
||||
pub(crate) fn install(job_id: Uuid) -> Self {
|
||||
let state = Arc::new(ManualTransitionJobCasBarrierState {
|
||||
job_id,
|
||||
paused: std::sync::atomic::AtomicBool::new(false),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
});
|
||||
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison");
|
||||
assert!(
|
||||
slot.is_none(),
|
||||
"manual transition job CAS barrier must be installed by one test at a time"
|
||||
);
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
if self.state.paused.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
arrived.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("manual transition job update should reach the deterministic CAS barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.add_permits(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ManualTransitionJobCasBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_manual_transition_job_before_first_cas(job_id: Uuid) {
|
||||
let barrier = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.job_id == job_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier
|
||||
&& barrier
|
||||
.paused
|
||||
.compare_exchange(false, true, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
barrier.arrived.notify_one();
|
||||
barrier
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("manual transition job CAS barrier should remain open")
|
||||
.forget();
|
||||
}
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
@@ -148,7 +246,6 @@ impl ManualTransitionJobRecord {
|
||||
|
||||
pub fn fail(&mut self, error: impl Into<String>) {
|
||||
self.state = ManualTransitionJobState::Failed;
|
||||
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
|
||||
self.error = Some(error.into());
|
||||
self.mark_updated_terminal();
|
||||
}
|
||||
@@ -662,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord {
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultRecord {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||
Self::new_with_reason(job_id, task_key, result, None)
|
||||
}
|
||||
@@ -1040,7 +1141,7 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
}
|
||||
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
|
||||
let data = job.encode().map_err(manual_transition_job_store_error)?;
|
||||
config_boundary::save_config_with_opts(
|
||||
config_boundary::save_config_with_opts_quiet(
|
||||
api,
|
||||
&object,
|
||||
data,
|
||||
@@ -1056,6 +1157,54 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Applies a job-record mutation with optimistic concurrency control.
|
||||
///
|
||||
/// The mutation returns whether the record needs to be persisted. When a lease
|
||||
/// is supplied, ownership is checked again after every conflicting write.
|
||||
pub async fn update_manual_transition_job_record<F>(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
update: F,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord>
|
||||
where
|
||||
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
|
||||
{
|
||||
update_manual_transition_job_record_from(api, job_id, expected_lease_id, None, update).await
|
||||
}
|
||||
|
||||
async fn update_manual_transition_job_record_from<F>(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
mut current: Option<(ManualTransitionJobRecord, String)>,
|
||||
mut update: F,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord>
|
||||
where
|
||||
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
|
||||
{
|
||||
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
|
||||
let (mut record, etag) = match current.take() {
|
||||
Some(current) => current,
|
||||
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
|
||||
};
|
||||
if expected_lease_id.is_some_and(|lease_id| record.lease_id != lease_id) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if !update(&mut record) {
|
||||
return Ok(record);
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_manual_transition_job_before_first_cas(job_id).await;
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => return Ok(record),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_manual_transition_worker_result_if_absent(
|
||||
api: Arc<ECStore>,
|
||||
record: &ManualTransitionWorkerResultRecord,
|
||||
@@ -1112,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_task_record(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1175,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_worker_result_stats(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1310,103 +1467,121 @@ async fn scan_manual_transition_worker_result_journal(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn reconcile_manual_transition_worker_results(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
reconcile_manual_transition_worker_results_inner(api, job_id, None, queue_snapshot, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_manual_transition_worker_results_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, false).await
|
||||
}
|
||||
|
||||
async fn reconcile_manual_transition_worker_results_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
mark_missing_results_unknown: bool,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let task_stats = match scan_manual_transition_task_journal(api.clone(), job_id).await? {
|
||||
ManualTransitionTaskJournal::Stats(stats) => stats,
|
||||
ManualTransitionTaskJournal::Corrupt(error) => {
|
||||
return mark_manual_transition_job_unknown_for_task_journal_error(api, job_id, error, queue_snapshot).await;
|
||||
return mark_manual_transition_job_unknown_for_task_journal_error(
|
||||
api,
|
||||
job_id,
|
||||
expected_lease_id,
|
||||
error,
|
||||
queue_snapshot,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let stats = match scan_manual_transition_worker_result_journal(api.clone(), job_id).await? {
|
||||
ManualTransitionWorkerResultJournal::Stats(stats) => stats,
|
||||
ManualTransitionWorkerResultJournal::Corrupt(error) => {
|
||||
return mark_manual_transition_job_unknown_for_worker_result_journal_error(api, job_id, error, queue_snapshot).await;
|
||||
return mark_manual_transition_job_unknown_for_worker_result_journal_error(
|
||||
api,
|
||||
job_id,
|
||||
expected_lease_id,
|
||||
error,
|
||||
queue_snapshot,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
let changed = record.apply_worker_result_counts(
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
let counts_changed = record.apply_worker_result_counts(
|
||||
stats.stats.completed,
|
||||
stats.stats.failed,
|
||||
&stats.stats.tier_failure_by_reason,
|
||||
task_stats.queued,
|
||||
queue_snapshot,
|
||||
);
|
||||
if !changed {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(
|
||||
api.clone(),
|
||||
&record.scope_key,
|
||||
record.job_id,
|
||||
record.lease_id,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let became_unknown = mark_missing_results_unknown && record.mark_unknown_if_worker_results_lost(queue_snapshot);
|
||||
changed = counts_changed || became_unknown;
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if !changed {
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn mark_manual_transition_job_unknown_for_task_journal_error(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
error: String,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if !record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot) {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
|
||||
.await?;
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
changed = record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot);
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if changed && record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn mark_manual_transition_job_unknown_for_worker_result_journal_error(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
error: String,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if !record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot) {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
|
||||
.await?;
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
changed = record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot);
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if changed && record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
@@ -1603,19 +1778,14 @@ async fn find_active_legacy_manual_transition_scope_conflict(
|
||||
}
|
||||
|
||||
pub async fn request_manual_transition_job_cancel(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
update_manual_transition_job_record(api, job_id, None, |record| {
|
||||
if record.is_terminal() || record.cancel_requested {
|
||||
return Ok(record);
|
||||
return false;
|
||||
}
|
||||
record.mark_cancel_requested();
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => return Ok(record),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
true
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn persist_manual_transition_job_progress(
|
||||
@@ -1624,10 +1794,39 @@ pub async fn persist_manual_transition_job_progress(
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
record.update_running_progress(report.clone(), queue_snapshot);
|
||||
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
persist_manual_transition_job_progress_inner(api, job_id, current.0.lease_id, Some(current), report, queue_snapshot).await
|
||||
}
|
||||
|
||||
pub async fn persist_manual_transition_job_progress_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
persist_manual_transition_job_progress_inner(api, job_id, expected_lease_id, None, report, queue_snapshot).await
|
||||
}
|
||||
|
||||
async fn persist_manual_transition_job_progress_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
current: Option<(ManualTransitionJobRecord, String)>,
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let record = update_manual_transition_job_record_from(api.clone(), job_id, Some(expected_lease_id), current, |record| {
|
||||
if record.state != ManualTransitionJobState::Running {
|
||||
return false;
|
||||
}
|
||||
record.update_running_progress(report.clone(), queue_snapshot);
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
if record.state == ManualTransitionJobState::Running {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -1661,25 +1860,58 @@ pub async fn renew_manual_transition_job_lease(
|
||||
job_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let (mut record, mut etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.state == ManualTransitionJobState::Running {
|
||||
if record.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
|
||||
record = reconcile_manual_transition_worker_results(api.clone(), job_id, queue_snapshot).await?;
|
||||
if record.is_terminal() || !record.report.worker_transition_pending() {
|
||||
return Ok(record);
|
||||
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
renew_manual_transition_job_lease_inner(api, job_id, current.0.lease_id, Some(current), queue_snapshot).await
|
||||
}
|
||||
|
||||
pub async fn renew_manual_transition_job_lease_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
renew_manual_transition_job_lease_inner(api, job_id, expected_lease_id, None, queue_snapshot).await
|
||||
}
|
||||
|
||||
async fn renew_manual_transition_job_lease_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
current: Option<(ManualTransitionJobRecord, String)>,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let (current, current_etag) = match current {
|
||||
Some(current) => current,
|
||||
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
|
||||
};
|
||||
if current.lease_id != expected_lease_id {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if current.state != ManualTransitionJobState::Running {
|
||||
return Ok(current);
|
||||
}
|
||||
if current.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
|
||||
return reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, true)
|
||||
.await;
|
||||
}
|
||||
let record = update_manual_transition_job_record_from(
|
||||
api.clone(),
|
||||
job_id,
|
||||
Some(expected_lease_id),
|
||||
Some((current, current_etag)),
|
||||
|record| {
|
||||
if record.state != ManualTransitionJobState::Running {
|
||||
return false;
|
||||
}
|
||||
(record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
}
|
||||
let became_terminal = record.mark_unknown_if_worker_results_lost(queue_snapshot);
|
||||
if !became_terminal {
|
||||
record.renew_lease(queue_snapshot);
|
||||
}
|
||||
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
|
||||
if became_terminal {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
true
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else if record.state == ManualTransitionJobState::Running {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
@@ -1688,15 +1920,31 @@ async fn renew_manual_transition_scope_admission_from_job(
|
||||
api: Arc<ECStore>,
|
||||
record: &ManualTransitionJobRecord,
|
||||
) -> EcstoreResult<()> {
|
||||
if let Ok((admission, admission_etag)) =
|
||||
load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await
|
||||
&& admission.job_id == record.job_id
|
||||
&& admission.lease_id == record.lease_id
|
||||
{
|
||||
let renewed_admission = ManualTransitionScopeAdmission::from_job(record);
|
||||
save_manual_transition_scope_admission_if_current(api, &renewed_admission, &admission_etag).await?;
|
||||
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
|
||||
let (admission, admission_etag) =
|
||||
match load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await {
|
||||
Ok(admission) => admission,
|
||||
Err(Error::ConfigNotFound) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if admission.job_id != record.job_id || admission.lease_id != record.lease_id {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let mut renewed_admission = ManualTransitionScopeAdmission::from_job(record);
|
||||
renewed_admission.lease_expires_at_unix_nanos = renewed_admission
|
||||
.lease_expires_at_unix_nanos
|
||||
.max(admission.lease_expires_at_unix_nanos);
|
||||
renewed_admission.updated_at_unix_nanos = renewed_admission.updated_at_unix_nanos.max(admission.updated_at_unix_nanos);
|
||||
if renewed_admission == admission {
|
||||
return Ok(());
|
||||
}
|
||||
match save_manual_transition_scope_admission_if_current(api.clone(), &renewed_admission, &admission_etag).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub async fn delete_manual_transition_scope_admission_if_current(
|
||||
@@ -2386,14 +2634,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_failure_counts_tier_failure() {
|
||||
fn manual_transition_job_record_control_plane_failure_does_not_count_tier_failure() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
|
||||
record.fail("missing tier");
|
||||
|
||||
assert_eq!(record.state, ManualTransitionJobState::Failed);
|
||||
assert_eq!(record.report.tier_failure, 1);
|
||||
assert_eq!(record.report.tier_failure, 0);
|
||||
assert_eq!(record.error.as_deref(), Some("missing tier"));
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
@@ -26,16 +27,37 @@ pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
async fn get_authoritative_metadata(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let metadata = sys.get_authoritative_metadata(bucket).await?;
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}")));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) async fn lifecycle_expiry_allowed(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<bool> {
|
||||
Ok(!get_authoritative_metadata(api, bucket, bucket_incarnation_id)
|
||||
.await?
|
||||
.table_bucket_enabled())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() {
|
||||
return Err(Error::other("persisted bucket lifecycle configuration is invalid"));
|
||||
@@ -51,6 +73,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
});
|
||||
}
|
||||
let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? {
|
||||
@@ -65,6 +88,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle,
|
||||
object_lock,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -125,6 +149,7 @@ mod tests {
|
||||
let lifecycle = lifecycle_config();
|
||||
metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap();
|
||||
metadata.lifecycle_config = Some(lifecycle);
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -132,7 +157,14 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some());
|
||||
let configs = get_expiry_configs(&store_a, bucket).await.unwrap();
|
||||
assert!(configs.lifecycle.is_some());
|
||||
assert!(configs.table_bucket_enabled);
|
||||
assert!(
|
||||
!lifecycle_expiry_allowed(&store_a, bucket, configs.bucket_incarnation_id)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,25 +15,35 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||
obj.version_purge_status.is_pending()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,10 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||
}
|
||||
|
||||
@@ -331,6 +331,10 @@ where
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -148,6 +148,10 @@ struct RecoveryCursor {
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn recover_tier_free_versions(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
|
||||
@@ -385,6 +385,10 @@ impl ExpiryOp for Jentry {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||
if let Err(err) = &result
|
||||
@@ -395,6 +399,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
#[cfg(test)]
|
||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||
@@ -405,6 +413,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -485,6 +497,10 @@ pub enum RemoteTierDeleteOutcome {
|
||||
AlreadyRemoved,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
|
||||
@@ -50,8 +50,16 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionTransactionError {
|
||||
#[error("transition transaction already exists")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
AlreadyExists,
|
||||
#[error("transition transaction is not found")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
NotFound,
|
||||
#[error("transition transaction is corrupt: {0}")]
|
||||
Corrupt(&'static str),
|
||||
|
||||
@@ -425,6 +425,15 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata for a physically new user bucket. Existing or fabricated legacy
|
||||
/// metadata must use [`Self::new`] so upgrades do not rewrite their
|
||||
/// durability posture.
|
||||
pub fn new_with_default_durability(name: &str) -> Self {
|
||||
let mut metadata = Self::new(name);
|
||||
metadata.durability_config_json = super::durability::new_bucket_durability_config_json();
|
||||
metadata
|
||||
}
|
||||
|
||||
pub fn save_file_path(&self) -> String {
|
||||
format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE)
|
||||
}
|
||||
@@ -1302,7 +1311,7 @@ mod test {
|
||||
assert!(bm.object_locking(), "object lock active via parsed config");
|
||||
}
|
||||
|
||||
/// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's
|
||||
/// backlog#580: KNOWN GAP (flagged 2026-03-06: "inline_data 前缀不同"). RustFS's
|
||||
/// inline-data extraction does not yet recover the object body from a
|
||||
/// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data`
|
||||
/// returns bytes that are not the `.metadata.bin` blob (no `format|version`
|
||||
@@ -1310,7 +1319,7 @@ mod test {
|
||||
/// inline-data framing is handled on the read path.
|
||||
/// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata
|
||||
/// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]`
|
||||
/// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that
|
||||
/// (the "`inline_data` 前缀不同" gap flagged on 2026-03-06 is that
|
||||
/// bitrot prefix, not a format incompatibility). Running the raw inline shard
|
||||
/// through RustFS's `BitrotReader` with the default `HighwayHash256S` must
|
||||
/// verify the checksum and yield the exact `.metadata.bin` blob.
|
||||
@@ -1378,6 +1387,43 @@ mod test {
|
||||
assert_ne!(old.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_bucket_metadata_constructor_does_not_seed_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new("legacy-or-fabricated");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_seeds_default_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("new-user-bucket");
|
||||
assert_eq!(
|
||||
metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
|
||||
let encoded = metadata.marshal_msg().expect("marshal metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("unmarshal metadata");
|
||||
assert_eq!(decoded.durability_config_json, metadata.durability_config_json);
|
||||
assert_eq!(
|
||||
decoded.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_can_inherit_global_durability() {
|
||||
temp_env::with_var(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"), || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("strict-fleet-new-bucket");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_config_updates_cannot_replace_bucket_incarnation() {
|
||||
let mut metadata = BucketMetadata::new("site-replication-update");
|
||||
|
||||
@@ -50,6 +50,75 @@ use uuid::Uuid;
|
||||
|
||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
struct ConfigWriteLockProbeState {
|
||||
bucket: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub struct ConfigWriteLockProbe {
|
||||
state: Arc<ConfigWriteLockProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl ConfigWriteLockProbe {
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(ConfigWriteLockProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
assert!(
|
||||
!probes.iter().any(|current| current.bucket == state.bucket),
|
||||
"config write lock probe must be unique for a bucket"
|
||||
);
|
||||
probes.push(Arc::clone(&state));
|
||||
drop(probes);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub async fn wait_until_attempted(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("bucket config update should attempt the transaction lock");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for ConfigWriteLockProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
probes.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_config_write_lock_attempt(bucket: &str) {
|
||||
let probe = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison")
|
||||
.iter()
|
||||
.find(|probe| probe.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(probe) = probe {
|
||||
probe.arrived.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MetadataLoadMode {
|
||||
Initial,
|
||||
@@ -288,6 +357,13 @@ pub(crate) fn bucket_metadata_sys_of(ctx: &crate::runtime::instance::InstanceCon
|
||||
get_bucket_metadata_sys()
|
||||
}
|
||||
|
||||
pub(crate) fn require_bucket_metadata_sys_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
) -> Result<Arc<RwLock<BucketMetadataSys>>> {
|
||||
ctx.bucket_metadata_sys()
|
||||
.ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
Ok(sys.read().await.api.clone())
|
||||
@@ -376,6 +452,15 @@ pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<Of
|
||||
Box::pin(update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
@@ -574,6 +659,41 @@ pub async fn update_under_transaction_lock(
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||
pub async fn delete_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||
}
|
||||
|
||||
pub async fn update_quota_if_incarnation(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||
sys.clone(),
|
||||
bucket,
|
||||
Some(expected_incarnation_id),
|
||||
))
|
||||
.await?;
|
||||
if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
|
||||
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
bucket: bucket.to_string(),
|
||||
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
@@ -688,6 +808,14 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||
}
|
||||
|
||||
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
||||
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -718,7 +846,26 @@ async fn acquire_transaction_lock_with_sys(
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
tokio::pin!(acquire);
|
||||
let mut notified = false;
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if !notified {
|
||||
notify_config_write_lock_attempt(bucket);
|
||||
notified = true;
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await?;
|
||||
Ok(guard)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
Ok(acquire.await?)
|
||||
}
|
||||
|
||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||
@@ -746,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -873,6 +1024,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read the quota configuration and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the bucket metadata
|
||||
/// transaction read lock.
|
||||
pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
metadata.quota_config_updated_at,
|
||||
))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -929,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1442,6 +1628,7 @@ impl BucketMetadataSys {
|
||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||
/// instead of supplied up front. Loads through this system's own store so
|
||||
/// the read and the persisted write target the same instance.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
@@ -1546,6 +1733,7 @@ impl BucketMetadataSys {
|
||||
/// A miss is never published as an authoritative default, and a snapshot
|
||||
/// read before delete plus same-name recreation cannot replace the new
|
||||
/// generation.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
|
||||
@@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
||||
|
||||
/// Check if an object has legal hold enabled.
|
||||
/// Returns true if legal hold is ON.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||
@@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
||||
/// # Returns
|
||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||
/// * `false` if the object is not locked
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||
// Delete markers are never locked
|
||||
if is_delete_marker {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use super::metadata_sys::get_bucket_metadata_sys;
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
|
||||
pub struct PolicySys {}
|
||||
@@ -27,6 +28,10 @@ impl PolicySys {
|
||||
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
|
||||
}
|
||||
|
||||
pub async fn try_is_allowed_for_store(store: &ECStore, args: &BucketPolicyArgs<'_>) -> Result<bool> {
|
||||
Self::is_allowed_with_policy(args, store.get_bucket_policy(args.bucket).await.map(|(policy, _)| policy)).await
|
||||
}
|
||||
|
||||
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
|
||||
match policy {
|
||||
Ok(policy) => Ok(policy.is_allowed(args).await),
|
||||
|
||||
@@ -52,6 +52,7 @@ impl QuotaChecker {
|
||||
) -> Result<QuotaCheckResult, QuotaError> {
|
||||
let start_time = Instant::now();
|
||||
let quota_config = self.get_quota_config(bucket).await?;
|
||||
let uses_durable_reservations = quota_config.uses_durable_reservations();
|
||||
|
||||
// If no quota limit is set, allow operation
|
||||
let quota_limit = match quota_config.quota {
|
||||
@@ -67,6 +68,7 @@ impl QuotaChecker {
|
||||
quota_limit: None,
|
||||
operation_size,
|
||||
remaining: None,
|
||||
uses_durable_reservations,
|
||||
});
|
||||
}
|
||||
Some(q) => q,
|
||||
@@ -74,14 +76,17 @@ impl QuotaChecker {
|
||||
|
||||
let current_usage = self.get_real_time_usage(bucket).await?;
|
||||
|
||||
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
|
||||
let expected_usage = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
current_usage.saturating_add(admission_size)
|
||||
}
|
||||
QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
|
||||
};
|
||||
|
||||
let allowed = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
quota_config.check_operation_allowed(current_usage, operation_size)
|
||||
quota_config.check_operation_allowed(current_usage, admission_size)
|
||||
}
|
||||
QuotaOperation::DeleteObject => true,
|
||||
};
|
||||
@@ -105,6 +110,7 @@ impl QuotaChecker {
|
||||
quota_limit: Some(quota_limit),
|
||||
operation_size,
|
||||
remaining,
|
||||
uses_durable_reservations,
|
||||
};
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
@@ -158,6 +164,26 @@ impl QuotaChecker {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn set_durable_quota_config_if_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
quota: BucketQuota,
|
||||
expected_incarnation_id: uuid::Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime, QuotaError> {
|
||||
let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig {
|
||||
reason: format!("Failed to serialize quota config: {}", e),
|
||||
})?;
|
||||
let start_time = Instant::now();
|
||||
let updated_at =
|
||||
crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
|
||||
.await
|
||||
.map_err(QuotaError::StorageError)?;
|
||||
|
||||
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
async fn set_quota_config_for_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
@@ -355,6 +381,7 @@ mod tests {
|
||||
quota_limit: None,
|
||||
operation_size: 1024,
|
||||
remaining: None,
|
||||
uses_durable_reservations: false,
|
||||
};
|
||||
|
||||
assert!(result.allowed);
|
||||
@@ -378,4 +405,13 @@ mod tests {
|
||||
let allowed = quota.check_operation_allowed(512, 1024);
|
||||
assert!(!allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
|
||||
let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
|
||||
let durable = BucketQuota::new(Some(5));
|
||||
|
||||
assert!(!legacy.check_operation_allowed(4, 2));
|
||||
assert!(durable.uses_durable_reservations());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,40 +13,100 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod checker;
|
||||
pub(crate) mod reservation;
|
||||
|
||||
use crate::error::Result;
|
||||
use rustfs_config::{
|
||||
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
|
||||
QUOTA_NOT_FOUND_ERROR_CODE,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||
use thiserror::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub enum QuotaType {
|
||||
/// Hard quota: reject immediately when exceeded
|
||||
/// Hard quota accounting.
|
||||
#[default]
|
||||
#[serde(alias = "HARD", alias = "hard")]
|
||||
Hard,
|
||||
}
|
||||
|
||||
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
|
||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
pub struct BucketQuota {
|
||||
#[serde(default)]
|
||||
pub quota: Option<u64>,
|
||||
/// Defaults to Hard when missing.
|
||||
#[serde(default)]
|
||||
pub quota_type: QuotaType,
|
||||
/// Optional durable reservation protocol. The wire format gives older
|
||||
/// nodes a zero hard quota so a mixed-version fleet fails closed.
|
||||
pub reservation_protocol: Option<u32>,
|
||||
/// Timestamp when this quota configuration was set (for audit purposes)
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
pub created_at: Option<OffsetDateTime>,
|
||||
/// Accept updated_at for compatibility; not used.
|
||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct BucketQuotaWire {
|
||||
#[serde(default)]
|
||||
quota: Option<u64>,
|
||||
#[serde(default)]
|
||||
quota_type: QuotaType,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_protocol: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_quota: Option<u64>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
created_at: Option<OffsetDateTime>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl Serialize for BucketQuota {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let durable = self.uses_durable_reservations();
|
||||
BucketQuotaWire {
|
||||
quota: if durable { Some(0) } else { self.quota },
|
||||
quota_type: self.quota_type.clone(),
|
||||
reservation_protocol: self.reservation_protocol,
|
||||
reservation_quota: if durable { self.quota } else { None },
|
||||
created_at: self.created_at,
|
||||
updated_at: self.updated_at,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BucketQuota {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||
Some(
|
||||
wire.reservation_quota
|
||||
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
|
||||
)
|
||||
} else {
|
||||
wire.quota
|
||||
};
|
||||
Ok(Self {
|
||||
quota,
|
||||
quota_type: wire.quota_type,
|
||||
reservation_protocol: wire.reservation_protocol,
|
||||
created_at: wire.created_at,
|
||||
updated_at: wire.updated_at,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketQuota {
|
||||
/// Serialize to JSON bytes. Same format as parse_all_configs.
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
@@ -63,6 +123,7 @@ impl BucketQuota {
|
||||
Self {
|
||||
quota,
|
||||
quota_type: QuotaType::Hard,
|
||||
reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
|
||||
created_at: Some(now),
|
||||
updated_at: None,
|
||||
}
|
||||
@@ -72,7 +133,19 @@ impl BucketQuota {
|
||||
self.quota
|
||||
}
|
||||
|
||||
pub fn uses_durable_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||
self.reservation_protocol
|
||||
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||
if operation_size == 0 {
|
||||
return true;
|
||||
}
|
||||
if let Some(quota_limit) = self.quota {
|
||||
current_usage.saturating_add(operation_size) <= quota_limit
|
||||
} else {
|
||||
@@ -94,6 +167,7 @@ pub struct QuotaCheckResult {
|
||||
pub quota_limit: Option<u64>,
|
||||
pub operation_size: u64,
|
||||
pub remaining: Option<u64>,
|
||||
pub uses_durable_reservations: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -119,6 +193,7 @@ pub enum QuotaError {
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub struct QuotaErrorResponse {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: String,
|
||||
@@ -134,6 +209,7 @@ pub struct QuotaErrorResponse {
|
||||
}
|
||||
|
||||
impl QuotaErrorResponse {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||
match quota_error {
|
||||
QuotaError::QuotaExceeded { .. } => Self {
|
||||
@@ -210,7 +286,59 @@ mod tests {
|
||||
let buf = q.marshal_msg().expect("marshal");
|
||||
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
|
||||
assert_eq!(q.quota, restored.quota);
|
||||
assert_eq!(q.quota_type, restored.quota_type);
|
||||
assert_eq!(restored.quota_type, QuotaType::Hard);
|
||||
assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clearing_quota_keeps_the_legacy_compatible_type() {
|
||||
let quota = BucketQuota::new(None);
|
||||
|
||||
assert_eq!(quota.quota_type, QuotaType::Hard);
|
||||
assert_eq!(quota.reservation_protocol, None);
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_quota_makes_legacy_nodes_fail_closed() {
|
||||
let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
|
||||
let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
|
||||
assert!(quota.uses_durable_reservations());
|
||||
assert_eq!(quota.quota, Some(2048));
|
||||
|
||||
#[derive(Deserialize)]
|
||||
enum LegacyQuotaType {
|
||||
Hard,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
#[allow(dead_code)]
|
||||
quota: Option<u64>,
|
||||
#[allow(dead_code)]
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
.expect("legacy readers should ignore the reservation protocol field");
|
||||
assert_eq!(legacy.quota, Some(0));
|
||||
assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||
let quota: BucketQuota =
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||
.expect("future protocol should remain parseable");
|
||||
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
assert!(quota.has_unsupported_reservation_protocol());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reservation_protocol_v1_requires_reservation_quota() {
|
||||
let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
|
||||
.expect_err("v1 without its authoritative quota must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("reservation_quota is required"));
|
||||
}
|
||||
|
||||
/// unmarshal accepts format without quota_type
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{
|
||||
pub(crate) use replication_filemeta_boundary::{
|
||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
};
|
||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
@@ -81,6 +81,6 @@ pub use replication_queue_boundary::{
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -37,6 +37,10 @@ impl ReplicationConfigStore {
|
||||
com::read_config_limited(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
|
||||
@@ -24,15 +24,27 @@ use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(
|
||||
config: &ReplicationLifecycleConfig,
|
||||
object_name: &str,
|
||||
@@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge {
|
||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: &ObjectToDelete,
|
||||
@@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge {
|
||||
check_replicate_delete(bucket, object, source, opts, None).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||
let pending_status = decision.pending_status();
|
||||
ReplicationState {
|
||||
|
||||
@@ -19,17 +19,33 @@ use time::OffsetDateTime;
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::msgp_decode;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) struct ReplicationMsgpCodec;
|
||||
|
||||
impl ReplicationMsgpCodec {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||
msgp_decode::read_msgp_ext8_time(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||
msgp_decode::skip_msgp_value(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||
msgp_decode::write_msgp_time(wr, time)
|
||||
}
|
||||
|
||||
@@ -77,6 +77,10 @@ impl ReplicationObjectBridge {
|
||||
load_delete_request_config_in(ctx, bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn delete_config_snapshot_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config(
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn load_delete_replication_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||
let mut tracker = DurableMrfBacklogTracker {
|
||||
available: true,
|
||||
@@ -712,6 +716,10 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
|
||||
// MRF worker lifecycle
|
||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
mrf_stop_tx: Sender<()>,
|
||||
|
||||
// Worker size tracking
|
||||
@@ -940,6 +948,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Resizes worker priority and counts
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn resize_worker_priority(
|
||||
&self,
|
||||
pri: ReplicationPriority,
|
||||
@@ -1180,6 +1192,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Queues an MRF save operation
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
@@ -1651,6 +1667,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling regular replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1664,6 +1684,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1678,6 +1702,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_mrf_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1691,6 +1719,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Delete resync metadata from replication resync state in memory
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||
let mut status_map = self.resyncer.status_map.write().await;
|
||||
status_map.remove(bucket);
|
||||
@@ -2567,6 +2599,12 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
|
||||
async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission;
|
||||
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission;
|
||||
async fn queue_replica_delete_batch(&self, deletes: &[DeletedObjectReplicationInfo]) -> ReplicationBatchAdmission;
|
||||
/// Persist one entry straight to the durable MRF journal, bypassing the
|
||||
/// live worker queues. For failures whose source state is already gone —
|
||||
/// e.g. exhausted delete-marker purges — where only a startup replay can
|
||||
/// retry, and live re-dispatch would loop unboundedly against a down
|
||||
/// target.
|
||||
async fn persist_mrf_entry(&self, entry: MrfReplicateEntry) -> ReplicationQueueAdmission;
|
||||
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
|
||||
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
|
||||
async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
|
||||
@@ -2607,6 +2645,10 @@ impl<S: ReplicationStorage> ReplicationPoolTrait for ReplicationPool<S> {
|
||||
self.queue_replica_delete_batch(deletes).await
|
||||
}
|
||||
|
||||
async fn persist_mrf_entry(&self, entry: MrfReplicateEntry) -> ReplicationQueueAdmission {
|
||||
self.queue_mrf_save_admission(entry, "delete_marker_purge").await
|
||||
}
|
||||
|
||||
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize) {
|
||||
self.resize(priority, max_workers, max_l_workers).await;
|
||||
}
|
||||
|
||||
@@ -21,11 +21,31 @@ pub(crate) use rustfs_replication::{
|
||||
should_count_head_proxy_failure,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||
|
||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||
|
||||
@@ -18,9 +18,10 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
use super::replication_filemeta_boundary::{
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
ReplicationWorkerOperation, VersionPurgeStatusType, get_replication_state, parse_replicate_decision,
|
||||
replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
|
||||
@@ -33,7 +34,7 @@ use super::replication_object_decision_boundary::{
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, should_retry_delete_marker_purge,
|
||||
};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
use super::replication_resync_boundary::{
|
||||
BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, encode_resync_file, is_version_id_mismatch,
|
||||
@@ -63,6 +64,7 @@ use futures::stream::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use metrics::counter;
|
||||
#[cfg(test)]
|
||||
use rmp_serde;
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -72,10 +74,10 @@ use rustfs_utils::http::{
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use tokio::io::AsyncRead;
|
||||
@@ -100,6 +102,10 @@ const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_s
|
||||
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
|
||||
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
|
||||
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -116,6 +122,10 @@ const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"tcp connect error",
|
||||
];
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
|
||||
|
||||
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
|
||||
@@ -181,6 +191,57 @@ fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
|
||||
should_count_head_proxy_failure(is_not_found, code, raw_status)
|
||||
}
|
||||
|
||||
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
|
||||
|
||||
/// Targets that already produced a version-identity-drift warning this
|
||||
/// process lifetime, by ARN. Deduping is advisory only (the metric still
|
||||
/// counts every drifting PUT), so a reconfigured target re-warning only
|
||||
/// after a restart is acceptable.
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
|
||||
|
||||
/// Runtime half of the P1-19 version-identity contract (the explicit probe
|
||||
/// lives in replication-check's VersionFidelity phase): every replication PUT
|
||||
/// response reveals whether the target adopted the source version id. A
|
||||
/// target minting its own ids silently breaks version-addressed deletes and
|
||||
/// heal, so surface it — once per target — instead of letting the divergence
|
||||
/// accumulate unseen.
|
||||
/// Pure drift judgment: the contract only applies when the source addressed a
|
||||
/// real (non-nil) version uuid, and drift means the target answered with
|
||||
/// anything else — including nothing at all.
|
||||
fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
|
||||
if source_version_id.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// A nil source uuid travels as the literal "null" (unversioned-source
|
||||
// semantics); no identity contract applies to it.
|
||||
if Uuid::parse_str(source_version_id).map(|uuid| uuid.is_nil()).unwrap_or(true) {
|
||||
return false;
|
||||
}
|
||||
assigned_version_id != Some(source_version_id)
|
||||
}
|
||||
|
||||
fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
}
|
||||
counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
|
||||
let mut warned = VERSION_IDENTITY_WARNED_ARNS
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if warned.insert(tgt_client.arn.clone()) {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
arn = %tgt_client.arn,
|
||||
endpoint = %tgt_client.endpoint,
|
||||
sent_version_id = %source_version_id,
|
||||
assigned_version_id = assigned_version_id.unwrap_or("<none>"),
|
||||
"Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
|
||||
if let Some(stats) = runtime_sources::replication_stats() {
|
||||
stats.inc_proxy(bucket, api, is_err).await;
|
||||
@@ -271,6 +332,10 @@ fn bounded_resync_max_jobs(value: usize) -> usize {
|
||||
#[derive(Debug)]
|
||||
pub struct ReplicationResyncer {
|
||||
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub worker_size: usize,
|
||||
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
|
||||
resync_admission: Arc<Semaphore>,
|
||||
@@ -487,6 +552,10 @@ impl ReplicationResyncer {
|
||||
.is_some_and(|status| status.failed_count > 0)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
@@ -1271,7 +1340,12 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
reason = "source_version_missing",
|
||||
"Skipping stale delete-marker replication"
|
||||
);
|
||||
return true;
|
||||
// The marker is gone at the source, but a replica of it may
|
||||
// already exist on the targets (a live race, or an MRF
|
||||
// purge-intent replay landing here on purpose). Purge instead
|
||||
// of just skipping; the result decides whether an MRF replay
|
||||
// may acknowledge the entry.
|
||||
return purge_stale_delete_marker_targets(&bucket, &dobj).await;
|
||||
}
|
||||
Err(err) => {
|
||||
source_state_verified = false;
|
||||
@@ -1485,29 +1559,6 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
|
||||
if requires_delayed_purge {
|
||||
let bucket_clone = bucket.clone();
|
||||
let dobj_clone = dobj.clone();
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..5 {
|
||||
if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
|
||||
&& source_delete_marker_missing(
|
||||
&*storage_clone,
|
||||
&bucket_clone,
|
||||
&dobj_clone.delete_object.object_name,
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if !is_version_purge {
|
||||
(
|
||||
@@ -1550,6 +1601,24 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
drs.replication_timestamp = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
if requires_delayed_purge {
|
||||
// Hand the watcher the MERGED replication state: `drs` folds this
|
||||
// round's per-target results into the previous state, including the
|
||||
// version ids the targets assigned to the markers they just created.
|
||||
// Spawning with the pre-merge `dobj` made the purge fall back to a
|
||||
// source-derived id, which a target that mints its own ids answers
|
||||
// with an idempotent 204 — the intent was then dropped while the
|
||||
// real marker stayed behind.
|
||||
let bucket_clone = bucket.clone();
|
||||
let mut dobj_clone = dobj.clone();
|
||||
dobj_clone.delete_object.replication_state = Some(drs.clone());
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
watch_and_purge_source_delete_marker(bucket_clone, dobj_clone, dsc_clone, storage_clone).await;
|
||||
});
|
||||
}
|
||||
|
||||
let event_name = if replication_status == ReplicationStatusType::Completed {
|
||||
EventName::ObjectReplicationComplete.to_string()
|
||||
} else {
|
||||
@@ -1608,12 +1677,36 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
};
|
||||
|
||||
replicate_delete_outcome(
|
||||
expected_targets,
|
||||
rinfos.targets.len(),
|
||||
state_persisted,
|
||||
source_state_verified,
|
||||
&replication_status,
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether a delete replication fully succeeded — the MRF replay acknowledges
|
||||
/// (drops) an entry exactly when this returns true.
|
||||
///
|
||||
/// The delayed purge is deliberately NOT an input: holding the outcome hostage
|
||||
/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker
|
||||
/// entry and retained them all in the durable MRF journal forever. Purge
|
||||
/// failures persist their own purge-intent entry instead
|
||||
/// (`watch_and_purge_source_delete_marker`), and replays of those entries
|
||||
/// report purge success through `purge_stale_delete_marker_targets`.
|
||||
fn replicate_delete_outcome(
|
||||
expected_targets: usize,
|
||||
replicated_targets: usize,
|
||||
state_persisted: bool,
|
||||
source_state_verified: bool,
|
||||
replication_status: &ReplicationStatusType,
|
||||
) -> bool {
|
||||
expected_targets > 0
|
||||
&& rinfos.targets.len() == expected_targets
|
||||
&& replicated_targets == expected_targets
|
||||
&& state_persisted
|
||||
&& source_state_verified
|
||||
&& !requires_delayed_purge
|
||||
&& replication_status == ReplicationStatusType::Completed
|
||||
&& *replication_status == ReplicationStatusType::Completed
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
@@ -1663,48 +1756,286 @@ fn delete_marker_purge_version_id(
|
||||
})
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
/// One purge pass over the eligible targets. Returns the ARNs that must be
|
||||
/// retried: the remote DELETE failed, or the target client was unavailable
|
||||
/// (e.g. a runtime cache miss). Inconsistent recorded version mappings are a
|
||||
/// deliberate refusal — retrying cannot make guessing a version id safe — so
|
||||
/// they are logged and excluded from the retry set.
|
||||
async fn replicate_delete_marker_purge_to_targets(
|
||||
bucket: &str,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
retry_arns: Option<&[String]>,
|
||||
) -> Vec<String> {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
let mut failed_arns = Vec::new();
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
if let Some(retry_arns) = retry_arns
|
||||
&& !retry_arns.iter().any(|arn| arn == &tgt_entry.arn)
|
||||
{
|
||||
continue;
|
||||
};
|
||||
|
||||
}
|
||||
// Decide the version first: refusing to guess is a per-target
|
||||
// FAILURE, not a silent skip. Reporting it as success would let the
|
||||
// watcher and the MRF replay drop the purge intent while the marker
|
||||
// is still on the target — the leak stays visible instead (the
|
||||
// entry is retained and keeps warning) until an operator repairs
|
||||
// the metadata.
|
||||
let Some(purge_version_id) = delete_marker_purge_version_id(
|
||||
dobj.delete_object.replication_state.as_ref(),
|
||||
&tgt_entry.arn,
|
||||
delete_marker_version_id,
|
||||
) else {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
"Skipping delete-marker purge: recorded target version metadata is inconsistent"
|
||||
reason = "recorded_target_version_inconsistent",
|
||||
"Delete-marker purge refused: recorded target version metadata is inconsistent"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "refused").increment(1);
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
let _ = tgt_client
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
reason = "target_client_missing",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "failed").increment(1);
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
match tgt_client
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
purge_version_id,
|
||||
replication_delete_marker_purge_remove_options(dobj.delete_object.delete_marker_mtime),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "purged").increment(1);
|
||||
}
|
||||
// The marker version is already gone on the target: the purge goal
|
||||
// is met. Strict S3 targets 404 here (RustFS/MinIO answer 204);
|
||||
// treating it as a failure would retain the intent entry forever.
|
||||
Err(error) if matches!(error.code.as_deref(), Some("NoSuchKey" | "NoSuchVersion")) => {
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "purged").increment(1);
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
error = %error,
|
||||
reason = "target_delete_failed",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "failed").increment(1);
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &error).await;
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
failed_arns
|
||||
}
|
||||
|
||||
const DELETE_MARKER_PURGE_WATCH_ROUNDS: usize = 5;
|
||||
const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_secs(1);
|
||||
|
||||
/// Watch the source delete marker for a short window after its replication.
|
||||
///
|
||||
/// KNOWN NON-DURABLE WINDOW: this task is detached, so a process exit inside
|
||||
/// the watch window loses an intent that has not been persisted yet. The
|
||||
/// window predates this code (the previous implementation had no durable
|
||||
/// channel at all, and no replay half either), so nothing regresses — closing
|
||||
/// it needs a write-ahead intent recorded before the parent delete is
|
||||
/// acknowledged, which is tracked as follow-up rather than done here: every
|
||||
/// delete-marker replication would pay a journal write for a purge that
|
||||
/// almost never happens.
|
||||
///
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
dobj: DeletedObjectReplicationInfo,
|
||||
dsc: ReplicateDecision,
|
||||
storage: Arc<S>,
|
||||
) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
};
|
||||
|
||||
// `pending` is None until the source marker is observed missing; after the
|
||||
// first purge pass it holds the targets that still need a successful purge.
|
||||
let mut pending: Option<Vec<String>> = None;
|
||||
for round in 0..DELETE_MARKER_PURGE_WATCH_ROUNDS {
|
||||
pending = match pending.take() {
|
||||
None => {
|
||||
if source_delete_marker_missing(&*storage, &bucket, &dobj.delete_object.object_name, delete_marker_version_id)
|
||||
.await
|
||||
{
|
||||
Some(replicate_delete_marker_purge_to_targets(&bucket, &dobj, &dsc, None).await)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(failed_arns) => Some(replicate_delete_marker_purge_to_targets(&bucket, &dobj, &dsc, Some(&failed_arns)).await),
|
||||
};
|
||||
if matches!(pending.as_deref(), Some([])) {
|
||||
return;
|
||||
}
|
||||
if round + 1 < DELETE_MARKER_PURGE_WATCH_ROUNDS {
|
||||
tokio::time::sleep(DELETE_MARKER_PURGE_WATCH_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
if let Some(failed_arns) = pending.filter(|failed_arns| !failed_arns.is_empty()) {
|
||||
enqueue_delete_marker_purge_mrf(&dobj, failed_arns).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Shape an exhausted purge intent as a marker-creation delete entry. Replay
|
||||
/// reconstructs it with `delete_marker: true`, finds the source marker gone,
|
||||
/// and funnels into the stale-marker branch of `replicate_delete_with_outcome`
|
||||
/// — which re-runs the purge without touching source state and reports purge
|
||||
/// success as the replay outcome.
|
||||
fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) -> MrfReplicateEntry {
|
||||
let mut entry = dobj.to_mrf_entry();
|
||||
entry.delete_marker = true;
|
||||
entry.version_id = None;
|
||||
entry.retry_count = 0;
|
||||
entry.target_arns = failed_arns;
|
||||
entry
|
||||
}
|
||||
|
||||
async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) {
|
||||
let arns = failed_arns.join(",");
|
||||
let miss_reason = match runtime_sources::replication_pool() {
|
||||
None => Some("replication_pool_unavailable"),
|
||||
Some(pool) => match pool.persist_mrf_entry(delete_marker_purge_mrf_entry(dobj, failed_arns)).await {
|
||||
ReplicationQueueAdmission::Queued => None,
|
||||
_ => Some("mrf_save_unavailable"),
|
||||
},
|
||||
};
|
||||
match miss_reason {
|
||||
None => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_MRF,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = dobj.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arns,
|
||||
state = "queued",
|
||||
"Delete-marker purge exhausted its watch window; intent persisted to the MRF journal"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "mrf_queued").increment(1);
|
||||
}
|
||||
Some(reason) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_MRF,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = dobj.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arns,
|
||||
state = "missed",
|
||||
reason,
|
||||
"Delete-marker purge intent could not be persisted for retry"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "mrf_missed").increment(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The marker vanished at the source while its replication was still pending
|
||||
/// (a live race), or this is an MRF purge-intent replay. Any marker already
|
||||
/// replicated to a target must still be purged; run bounded retry passes and
|
||||
/// report the result so an MRF replay only acknowledges the entry once every
|
||||
/// target is clean. Live callers persist a fresh purge intent on failure;
|
||||
/// replay callers (`ReplicationType::Heal`) rely on Missed retention instead,
|
||||
/// so the journal does not accumulate duplicate entries.
|
||||
///
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
/// entry instead.
|
||||
async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo) -> bool {
|
||||
let decision_str = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.map(|state| state.replicate_decision_str.clone())
|
||||
.unwrap_or_default();
|
||||
let dsc = match parse_replicate_decision(bucket, &decision_str) {
|
||||
Ok(dsc) => dsc,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
error = %error,
|
||||
reason = "replicate_decision_parse_failed",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let rounds = if dobj.op_type == ReplicationType::Heal {
|
||||
DELETE_MARKER_PURGE_WATCH_ROUNDS
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let mut failed_arns = replicate_delete_marker_purge_to_targets(bucket, dobj, &dsc, None).await;
|
||||
for _ in 1..rounds {
|
||||
if failed_arns.is_empty() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(DELETE_MARKER_PURGE_WATCH_INTERVAL).await;
|
||||
failed_arns = replicate_delete_marker_purge_to_targets(bucket, dobj, &dsc, Some(&failed_arns)).await;
|
||||
}
|
||||
if failed_arns.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if dobj.op_type != ReplicationType::Heal {
|
||||
enqueue_delete_marker_purge_mrf(dobj, failed_arns).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
|
||||
@@ -2605,6 +2936,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
audit_target_version_identity(
|
||||
&tgt_client,
|
||||
&put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
@@ -3012,6 +3350,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
audit_target_version_identity(
|
||||
&tgt_client,
|
||||
&put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
@@ -3203,21 +3548,34 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
|
||||
let actual_size = replication_multipart_complete_actual_size(&object_info.user_defined);
|
||||
|
||||
cli.complete_multipart_upload(
|
||||
dst_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
uploaded_parts,
|
||||
&replication_complete_multipart_options(actual_size, object_info.etag.clone().unwrap_or_default(), object_info.mod_time),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
let completed = cli
|
||||
.complete_multipart_upload(
|
||||
dst_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
uploaded_parts,
|
||||
&replication_complete_multipart_options(
|
||||
actual_size,
|
||||
object_info.etag.clone().unwrap_or_default(),
|
||||
object_info.mod_time,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
|
||||
// Multipart decides the target version at initiate time and only reveals
|
||||
// it on completion, so this is where the identity contract is observable
|
||||
// for this path. A target can mirror PutObject version ids and still mint
|
||||
// its own here, which would leave multipart deletes and heals addressing
|
||||
// a version that never existed.
|
||||
audit_target_version_identity(&cli, &put_opts.internal.source_version_id, completed.version_id());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
@@ -3257,6 +3615,27 @@ mod tests {
|
||||
ReplicationTargetStore::register_test_target(target).await;
|
||||
}
|
||||
|
||||
/// P1-19 runtime spot-check exemption matrix: drift only applies when the
|
||||
/// source addressed a real version uuid.
|
||||
#[test]
|
||||
fn test_version_identity_drift_judgment() {
|
||||
let source = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
|
||||
for (sent, got, expected) in [
|
||||
(source, Some(source), false),
|
||||
(source, Some("0e304ce5-33e9-4b8a-9b12-9e40a53e6ded"), true),
|
||||
(source, None, true),
|
||||
("", None, false),
|
||||
("null", Some("anything"), false),
|
||||
("00000000-0000-0000-0000-000000000000", Some("anything"), false),
|
||||
] {
|
||||
assert_eq!(
|
||||
version_identity_drifted(sent, got),
|
||||
expected,
|
||||
"sent {sent:?} got {got:?} must judge drift = {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_admission_configuration_is_bounded() {
|
||||
assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");
|
||||
@@ -3581,6 +3960,101 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// P1-21 regression guard for the outcome formula. A fully successful
|
||||
/// delete-marker replication must acknowledge its MRF entry: the formula
|
||||
/// once carried `&& !requires_delayed_purge`, which pinned every
|
||||
/// delete-marker entry to Missed and retained the whole backlog forever.
|
||||
/// (Deterministically staging a marker-creation entry in the durable
|
||||
/// journal from e2e would require saturating the worker queues, so the
|
||||
/// formula is pinned here instead; the purge-intent replay half is pinned
|
||||
/// by the delayed-purge e2e pair.)
|
||||
#[test]
|
||||
fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() {
|
||||
assert!(
|
||||
replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed),
|
||||
"a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending"
|
||||
);
|
||||
assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed));
|
||||
}
|
||||
|
||||
/// P1-21 review follow-up: a target whose recorded marker version is
|
||||
/// inconsistent must be reported as a per-target FAILURE. Treating the
|
||||
/// refusal as success let the watcher and the MRF replay drop the purge
|
||||
/// intent while the marker was still on the target.
|
||||
#[tokio::test]
|
||||
async fn test_delete_marker_purge_reports_corrupt_recorded_version_as_failure() {
|
||||
let arn = format!("arn:rustfs:replication:us-east-1:corrupt:{}", Uuid::new_v4());
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
dsc.set(ReplicateTargetDecision::new(arn.clone(), true, false));
|
||||
|
||||
let mut state = ReplicationState {
|
||||
target_delete_marker_version_ids_corrupt: true,
|
||||
..Default::default()
|
||||
};
|
||||
state.targets.insert(arn.clone(), ReplicationStatusType::Completed);
|
||||
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "doc.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(state),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// No target client is registered: the refusal must be decided from
|
||||
// the recorded metadata alone, before any remote call is attempted.
|
||||
let failed = replicate_delete_marker_purge_to_targets("bucket-a", &dobj, &dsc, None).await;
|
||||
|
||||
assert_eq!(
|
||||
failed,
|
||||
vec![arn],
|
||||
"a refused purge must stay in the failed set so the intent is never acknowledged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() {
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "doc.txt".to_string(),
|
||||
// A version-purge flavored source event: the entry must still
|
||||
// be reshaped as a marker-creation delete so replay funnels
|
||||
// into the stale-marker branch instead of re-running the full
|
||||
// delete replication (whose source-state stamping would fail
|
||||
// against the already-purged version).
|
||||
delete_marker: false,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
|
||||
|
||||
assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes");
|
||||
assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay");
|
||||
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
|
||||
assert_eq!(
|
||||
entry.target_arns,
|
||||
vec!["arn:a".to_string()],
|
||||
"only the targets whose purge failed may be retried"
|
||||
);
|
||||
assert_eq!(entry.retry_count, 0);
|
||||
assert_eq!(entry.bucket, "bucket-a");
|
||||
assert_eq!(entry.object, "doc.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
|
||||
@@ -340,6 +340,10 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Site replication update replica statistics
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn sr_update_replica_stat(&self, size: i64) {
|
||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -704,6 +708,12 @@ impl ReplicationStats {
|
||||
} else {
|
||||
BucketReplicationStats::new()
|
||||
};
|
||||
// Stamp the serializable failure windows from the live samples: the
|
||||
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
||||
// is what cluster aggregation and the metrics endpoints see.
|
||||
for stat in replication_stats.stats.values_mut() {
|
||||
stat.fail_stats.refresh_windows();
|
||||
}
|
||||
let uptime = if cache.contains_key(bucket) {
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||
SRMetricsSummary, XferStats,
|
||||
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
||||
};
|
||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
||||
// can project the internal stats onto the minio-go response shapes through
|
||||
// the storage_api facade chain.
|
||||
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
|
||||
@@ -27,8 +27,10 @@ use rustfs_utils::http::{
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
||||
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
||||
ssec_replication_transport_header,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
@@ -119,6 +121,27 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
||||
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
||||
let key = key.as_bytes();
|
||||
key.len() == prefix.len() + suffix.len()
|
||||
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
||||
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
||||
}
|
||||
|
||||
[
|
||||
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| {
|
||||
["x-rustfs-", "x-minio-"]
|
||||
.iter()
|
||||
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
@@ -176,6 +199,11 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_legacy_source_replication_timestamp_key(key) {
|
||||
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_internal_key(key) || is_standard_header(key) {
|
||||
continue;
|
||||
}
|
||||
@@ -259,15 +287,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
|
||||
if !tags.is_empty() {
|
||||
put_options.user_tags = tags;
|
||||
put_options.internal.tagging_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Load the stored tagging timestamp independently of whether any tags
|
||||
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||
// With no stored key, fall back to mod_time only while tags exist
|
||||
// (MinIO parity); a tagless object without the key was never tagged and
|
||||
// keeps the epoch default (no header).
|
||||
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||
{
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else if !put_options.user_tags.is_empty() {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
let metadata = &*object_info.user_defined;
|
||||
|
||||
@@ -283,13 +319,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
put_options.cache_control = cache_control.to_string();
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
if !retain_until_date.is_empty() {
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
}
|
||||
put_options.internal.retention_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
@@ -694,6 +732,110 @@ mod tests {
|
||||
assert!(options.internal.replication_request);
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||
/// still be loaded (and therefore sent) so the replica can order the
|
||||
/// deletion against concurrent tag edits.
|
||||
#[test]
|
||||
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
assert!(options.user_tags.is_empty());
|
||||
assert_eq!(
|
||||
options.internal.tagging_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||
"the stored tagging timestamp must load independently of remaining tags"
|
||||
);
|
||||
|
||||
// A tagless object without the stored key was never tagged: the epoch
|
||||
// default keeps the header unsent.
|
||||
let untagged = ObjectInfo {
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
||||
let legacy_keys = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(
|
||||
legacy_keys
|
||||
.iter()
|
||||
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
for legacy_key in legacy_keys {
|
||||
assert!(!options.user_metadata.contains_key(legacy_key));
|
||||
assert_eq!(
|
||||
options
|
||||
.user_metadata
|
||||
.get(&format!("x-amz-meta-{legacy_key}"))
|
||||
.map(String::as_str),
|
||||
Some("2099-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
||||
let mut metadata = HashMap::from([
|
||||
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
||||
]);
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
||||
|
||||
assert!(options.mode.is_none());
|
||||
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(
|
||||
options.internal.retention_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
||||
);
|
||||
let headers = options.header();
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
||||
Some("2026-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||
|
||||
@@ -40,7 +40,14 @@ impl ARN {
|
||||
|
||||
impl Display for ARN {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
// The `minio` partition is deliberate: madmin-go's ParseARN
|
||||
// hard-rejects any other partition, so native mc/madmin tooling can
|
||||
// only decode remote-target ARNs minted in this form (backlog#1675
|
||||
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
||||
// readable via the FromStr whitelist below; runtime matching between
|
||||
// targets and replication rules is by full-string equality, so mixed
|
||||
// partitions coexist safely.
|
||||
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +55,12 @@ impl FromStr for ARN {
|
||||
type Err = std::io::Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
if !s.starts_with("arn:rustfs:") {
|
||||
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
||||
// from_str(...).unwrap_or_default()` below never fails, so this is
|
||||
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
||||
// the legacy partition and must stay accepted forever (persisted
|
||||
// bucket-targets.json / replication configs from older releases).
|
||||
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||
}
|
||||
|
||||
@@ -101,14 +113,50 @@ mod tests {
|
||||
}
|
||||
|
||||
/// RustFS commonly generates ARNs with an empty region:
|
||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
||||
#[test]
|
||||
fn from_str_handles_empty_region_segment() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
||||
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
||||
/// native mc/madmin tooling cannot decode remote-target listings.
|
||||
#[test]
|
||||
fn display_emits_minio_partition() {
|
||||
let arn = ARN::new(
|
||||
BucketTargetType::ReplicationService,
|
||||
"depl-123".to_string(),
|
||||
String::new(),
|
||||
"bucket-a".to_string(),
|
||||
);
|
||||
|
||||
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
||||
}
|
||||
|
||||
/// Persisted bucket-targets.json files from older RustFS releases carry
|
||||
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
||||
#[test]
|
||||
fn from_str_accepts_legacy_rustfs_partition() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "us-east-1");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
||||
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
||||
/// would otherwise parse as `type=None`.
|
||||
#[test]
|
||||
fn from_str_rejects_unknown_partition() {
|
||||
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
||||
assert!(ARN::from_str("not-an-arn").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,10 @@ impl fmt::Debug for Credentials {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub enum ServiceType {
|
||||
#[default]
|
||||
Replication,
|
||||
|
||||
@@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
||||
if object_name.len() > 1024 {
|
||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
||||
}
|
||||
if !object_name.is_ascii() {
|
||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
||||
if object_name.trim().is_empty() {
|
||||
return Err(Error::other("Object name cannot be empty"));
|
||||
}
|
||||
check_valid_object_name_prefix(object_name)
|
||||
}
|
||||
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
@@ -100,6 +83,10 @@ where
|
||||
Ok(ans)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
@@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||
/// accept the full S3 key character set.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||
|
||||
/// Client-facing reason for path segments Windows can store but not address
|
||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||
|
||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||
@@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||
object.split(['/', '\\']).any(|segment| {
|
||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||
|
||||
@@ -90,6 +90,10 @@ impl BucketVersioningSys {
|
||||
/// caller's own instance context so a second in-process store never
|
||||
/// answers with the first instance's versioning state; falls back to the
|
||||
/// ambient system when the instance cell is not initialized.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||
return Ok(VersioningConfiguration::default());
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
use std::{
|
||||
@@ -655,6 +656,7 @@ async fn list_path_raw_inner(
|
||||
errs.push(None);
|
||||
}
|
||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -676,6 +678,21 @@ async fn list_path_raw_inner(
|
||||
let mut has_err = 0;
|
||||
let mut agree = 0;
|
||||
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cancel = &revjob_rx;
|
||||
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
||||
});
|
||||
for (i, outcome) in join_all(concurrent_peeks).await {
|
||||
peek_outcomes[i] = Some(outcome);
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
has_err += 1;
|
||||
@@ -685,7 +702,10 @@ async fn list_path_raw_inner(
|
||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||
entry
|
||||
} else {
|
||||
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
||||
let Some(outcome) = peek_outcomes[i].take() else {
|
||||
return Err(DiskError::Unexpected);
|
||||
};
|
||||
match outcome {
|
||||
PeekOutcome::Ready(res) => {
|
||||
if let Some(entry) = res {
|
||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||
@@ -1295,6 +1315,36 @@ mod tests {
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
||||
let peek_timeout = Duration::from_millis(20);
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None, None, None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
],
|
||||
peek_timeout: Some(peek_timeout),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("all stalled readers should fail the listing");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(
|
||||
started.elapsed(),
|
||||
peek_timeout,
|
||||
"reader deadlines must overlap instead of accumulating once per disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||
let entry = MetaCacheEntry {
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
if policy == "" {
|
||||
return self.remove_bucket_policy(bucket_name).await;
|
||||
}
|
||||
|
||||
self.put_bucket_policy(bucket_name, policy).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
|
||||
content_length: policy.len() as i64,
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_sha256_hex: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
//if resp != nil {
|
||||
if resp_status != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
if resp_status != StatusCode::NO_CONTENT {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
|
||||
Ok(bucket_policy)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let policy = String::from_utf8_lossy(&body_vec).to_string();
|
||||
Ok(policy)
|
||||
}
|
||||
}
|
||||
@@ -229,17 +229,6 @@ pub fn http_resp_to_error_response(
|
||||
err_resp
|
||||
}
|
||||
|
||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
||||
.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||
@@ -295,16 +284,6 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
||||
message: message.to_string(),
|
||||
request_id: "rustfs".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -135,6 +135,10 @@ impl Object {
|
||||
Self { ..Default::default() }
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||
let _ = request.did_offset_change;
|
||||
let _ = request.offset;
|
||||
@@ -150,12 +154,20 @@ impl Object {
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||
self.curr_offset += bytes_read;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||
let mut read_req = GetRequest {
|
||||
is_read_op: true,
|
||||
@@ -180,6 +192,10 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let _ = self.do_get_request(&GetRequest {
|
||||
@@ -192,6 +208,10 @@ impl Object {
|
||||
Ok(self.object_info.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||
self.curr_offset = offset;
|
||||
|
||||
@@ -219,6 +239,10 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let seek_req = GetRequest {
|
||||
@@ -253,6 +277,10 @@ impl Object {
|
||||
Ok(self.curr_offset)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||
self.is_closed = true;
|
||||
Ok(())
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::dto::Owner;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grantee {
|
||||
pub id: String,
|
||||
pub display_name: String,
|
||||
pub uri: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grant {
|
||||
pub grantee: Grantee,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct AccessControlList {
|
||||
pub grant: Vec<Grant>,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct AccessControlPolicy {
|
||||
#[serde(skip)]
|
||||
owner: Owner,
|
||||
pub access_control_list: AccessControlList,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_acl(&self, bucket_name: &str, object_name: &str) -> Result<ObjectInfo, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("acl".to_string(), "".to_string());
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
|
||||
let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(body_vec).unwrap()) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let mut obj_info = self
|
||||
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
|
||||
.await?;
|
||||
|
||||
obj_info.owner.display_name = res.owner.display_name.clone();
|
||||
obj_info.owner.id = res.owner.id.clone();
|
||||
|
||||
//obj_info.grant.extend(res.access_control_list.grant);
|
||||
|
||||
let canned_acl = get_canned_acl(&res);
|
||||
if canned_acl != "" {
|
||||
obj_info
|
||||
.metadata
|
||||
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
|
||||
return Ok(obj_info);
|
||||
}
|
||||
|
||||
let grant_acl = get_amz_grant_acl(&res);
|
||||
/*for (k, v) in grant_acl {
|
||||
obj_info.metadata.insert(HeaderName::from_bytes(k.as_bytes()).unwrap(), HeaderValue::from_str(&v.to_string()).unwrap());
|
||||
}*/
|
||||
|
||||
Ok(obj_info)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_canned_acl(ac_policy: &AccessControlPolicy) -> String {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
|
||||
if grants.len() == 1 {
|
||||
if grants[0].grantee.uri == "" && grants[0].permission == "FULL_CONTROL" {
|
||||
return "private".to_string();
|
||||
}
|
||||
} else if grants.len() == 2 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && &g.permission == "READ" {
|
||||
return "authenticated-read".to_string();
|
||||
}
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && &g.permission == "READ" {
|
||||
return "public-read".to_string();
|
||||
}
|
||||
if g.permission == "READ" && g.grantee.id == ac_policy.owner.id.clone().unwrap() {
|
||||
return "bucket-owner-read".to_string();
|
||||
}
|
||||
}
|
||||
} else if grants.len() == 3 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && g.permission == "WRITE" {
|
||||
return "public-read-write".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn get_amz_grant_acl(ac_policy: &AccessControlPolicy) -> HashMap<String, Vec<String>> {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
let mut res = HashMap::<String, Vec<String>>::new();
|
||||
|
||||
for g in grants {
|
||||
let mut id = "id=".to_string();
|
||||
id.push_str(&g.grantee.id);
|
||||
let permission: &str = &g.permission;
|
||||
match permission {
|
||||
"READ" => {
|
||||
res.entry("X-Amz-Grant-Read".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE" => {
|
||||
res.entry("X-Amz-Grant-Write".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"READ_ACP" => {
|
||||
res.entry("X-Amz-Grant-Read-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE_ACP" => {
|
||||
res.entry("X-Amz-Grant-Write-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"FULL_CONTROL" => {
|
||||
res.entry("X-Amz-Grant-Full-Control".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
|
||||
use crate::client::{
|
||||
api_get_object_acl::AccessControlPolicy,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::header::{X_AMZ_MAX_PARTS, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER, X_AMZ_VERSION_ID};
|
||||
|
||||
pub struct ObjectAttributesOptions {
|
||||
pub max_parts: i64,
|
||||
pub version_id: String,
|
||||
pub part_number_marker: i64,
|
||||
//server_side_encryption: encrypt::ServerSide,
|
||||
}
|
||||
|
||||
pub struct ObjectAttributes {
|
||||
pub version_id: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
pub object_attributes_response: ObjectAttributesResponse,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
version_id: "".to_string(),
|
||||
last_modified: OffsetDateTime::now_utc(),
|
||||
object_attributes_response: ObjectAttributesResponse::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct Checksum {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
}
|
||||
|
||||
impl Checksum {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
checksum_crc32: "".to_string(),
|
||||
checksum_crc32c: "".to_string(),
|
||||
checksum_sha1: "".to_string(),
|
||||
checksum_sha256: "".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectParts {
|
||||
pub parts_count: i64,
|
||||
pub part_number_marker: i64,
|
||||
pub next_part_number_marker: i64,
|
||||
pub max_parts: i64,
|
||||
is_truncated: bool,
|
||||
parts: Vec<ObjectAttributePart>,
|
||||
}
|
||||
|
||||
impl ObjectParts {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
parts_count: 0,
|
||||
part_number_marker: 0,
|
||||
next_part_number_marker: 0,
|
||||
max_parts: 0,
|
||||
is_truncated: false,
|
||||
parts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectAttributesResponse {
|
||||
pub etag: String,
|
||||
pub storage_class: String,
|
||||
pub object_size: i64,
|
||||
pub checksum: Checksum,
|
||||
pub object_parts: ObjectParts,
|
||||
}
|
||||
|
||||
impl ObjectAttributesResponse {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
etag: "".to_string(),
|
||||
storage_class: "".to_string(),
|
||||
object_size: 0,
|
||||
checksum: Checksum::new(),
|
||||
object_parts: ObjectParts::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
struct ObjectAttributePart {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
part_number: i64,
|
||||
size: i64,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
pub async fn parse_response(&mut self, h: &HeaderMap, body_vec: Vec<u8>) -> Result<(), std::io::Error> {
|
||||
let last_modified = h
|
||||
.get("Last-Modified")
|
||||
.ok_or_else(|| std::io::Error::other("missing Last-Modified header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified header: {e}")))?;
|
||||
let mod_time = OffsetDateTime::parse(last_modified, ISO8601_DATEFORMAT)
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified date: {e}")))?;
|
||||
self.last_modified = mod_time;
|
||||
|
||||
let version_id = h
|
||||
.get(X_AMZ_VERSION_ID)
|
||||
.ok_or_else(|| std::io::Error::other("missing version ID header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid version ID header: {e}")))?;
|
||||
self.version_id = version_id.to_string();
|
||||
|
||||
let body_str = String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 body: {e}")))?;
|
||||
let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&body_str) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
self.object_attributes_response = response;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_attributes(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: ObjectAttributesOptions,
|
||||
) -> Result<ObjectAttributes, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("attributes".to_string(), "".to_string());
|
||||
if opts.version_id != "" {
|
||||
url_values.insert("versionId".to_string(), opts.version_id);
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
X_AMZ_OBJECT_ATTRIBUTES,
|
||||
HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).expect("valid header value"),
|
||||
);
|
||||
|
||||
if opts.part_number_marker > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_PART_NUMBER_MARKER,
|
||||
HeaderValue::from_str(&opts.part_number_marker.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
if opts.max_parts > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&opts.max_parts.to_string()).expect("valid header value"),
|
||||
);
|
||||
} else {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
/*if opts.server_side_encryption.is_some() {
|
||||
opts.server_side_encryption.Marshal(headers);
|
||||
}*/
|
||||
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: headers,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
let has_etag = h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("");
|
||||
if !has_etag.is_empty() {
|
||||
return Err(std::io::Error::other(
|
||||
"get_object_attributes is not supported by the current endpoint version",
|
||||
));
|
||||
}
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
let err_body =
|
||||
String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 error body: {e}")))?;
|
||||
let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
return Err(std::io::Error::other(er.access_control_list.permission));
|
||||
}
|
||||
|
||||
let mut oa = ObjectAttributes::new();
|
||||
oa.parse_response(&h, body_vec).await?;
|
||||
|
||||
Ok(oa)
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[cfg(not(windows))]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
use tokio::fs::{self, OpenOptions};
|
||||
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, transition_api::TransitionClient,
|
||||
};
|
||||
|
||||
async fn prepare_download_target(file_path: &Path) -> io::Result<()> {
|
||||
match fs::metadata(file_path).await {
|
||||
Ok(metadata) if metadata.is_dir() => {
|
||||
return Err(io::Error::other(err_invalid_argument("filename is a directory.")));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
if let Some(parent) = file_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
fs::create_dir_all(parent).await?;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let mut permissions = fs::metadata(parent).await?.permissions();
|
||||
permissions.set_mode(0o700);
|
||||
fs::set_permissions(parent, permissions).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_part_path(file_path: &Path) -> PathBuf {
|
||||
PathBuf::from(format!("{}.part.rustfs", file_path.display()))
|
||||
}
|
||||
|
||||
async fn open_download_part_file(file_part_path: &Path) -> io::Result<tokio::fs::File> {
|
||||
let mut options = OpenOptions::new();
|
||||
options.create(true).truncate(false).read(true).write(true);
|
||||
|
||||
#[cfg(not(windows))]
|
||||
options.mode(0o600);
|
||||
|
||||
options.open(file_part_path).await
|
||||
}
|
||||
|
||||
async fn cleanup_part_file(file_part_path: &Path) {
|
||||
let _ = fs::remove_file(file_part_path).await;
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn fget_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
file_path: &str,
|
||||
mut opts: GetObjectOptions,
|
||||
) -> Result<(), io::Error> {
|
||||
let file_path = Path::new(file_path);
|
||||
prepare_download_target(file_path).await?;
|
||||
|
||||
let file_part_path = build_part_path(file_path);
|
||||
let mut file_part = open_download_part_file(&file_part_path).await?;
|
||||
let existing_len = file_part.metadata().await?.len();
|
||||
if existing_len > 0 {
|
||||
opts.set_range(existing_len as i64, 0)?;
|
||||
file_part.seek(SeekFrom::Start(existing_len)).await?;
|
||||
}
|
||||
|
||||
let (_object_info, _headers, mut object_reader) = self.get_object_inner(bucket_name, object_name, &opts).await?;
|
||||
if let Err(err) = tokio::io::copy(&mut object_reader, &mut file_part).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Err(err) = file_part.flush().await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
drop(file_part);
|
||||
|
||||
if let Err(err) = fs::rename(&file_part_path, file_path).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_allows_missing_file_and_creates_parent_dirs() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("nested").join("object.bin");
|
||||
|
||||
prepare_download_target(&target)
|
||||
.await
|
||||
.expect("missing target should be accepted");
|
||||
|
||||
assert!(target.parent().expect("parent").exists(), "parent directory should be created");
|
||||
assert!(
|
||||
fs::metadata(&target).await.is_err(),
|
||||
"preparing the target should not create the final file eagerly"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_rejects_directory_paths() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target_dir = dir.path().join("download-dir");
|
||||
fs::create_dir_all(&target_dir).await.expect("target dir");
|
||||
|
||||
let err = prepare_download_target(&target_dir)
|
||||
.await
|
||||
.expect_err("directory targets must be rejected");
|
||||
|
||||
assert!(err.to_string().contains("directory"), "unexpected error for directory target: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_download_part_file_creates_part_file() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("object.bin");
|
||||
let part_path = build_part_path(&target);
|
||||
|
||||
let file = open_download_part_file(&part_path)
|
||||
.await
|
||||
.expect("part file should be created");
|
||||
drop(file);
|
||||
|
||||
assert!(part_path.exists(), "part file should exist after creation");
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ use crate::client::{
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
||||
credentials::SignatureType,
|
||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||
|
||||
@@ -30,10 +30,6 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||
let unknown_size;
|
||||
let mut object_size = object_size;
|
||||
|
||||
@@ -81,18 +81,6 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UploadedPartRes {
|
||||
pub error: std::io::Error,
|
||||
pub part_num: i64,
|
||||
pub size: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
pub struct UploadPartReq {
|
||||
pub part_num: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object_multipart_stream(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{err_invalid_argument, http_resp_to_error_response},
|
||||
api_get_object_acl::AccessControlList,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::BufReader;
|
||||
|
||||
const TIER_STANDARD: &str = "Standard";
|
||||
const TIER_BULK: &str = "Bulk";
|
||||
const TIER_EXPEDITED: &str = "Expedited";
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Encryption {
|
||||
pub encryption_type: String,
|
||||
pub kms_context: String,
|
||||
pub kms_key_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct MetadataEntry {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
pub struct S3 {
|
||||
pub access_control_list: AccessControlList,
|
||||
pub bucket_name: String,
|
||||
pub prefix: String,
|
||||
pub canned_acl: String,
|
||||
pub encryption: Encryption,
|
||||
pub storage_class: String,
|
||||
//tagging: Tags,
|
||||
pub user_metadata: MetadataEntry,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn restore_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
version_id: &str,
|
||||
restore_req: &RestoreRequest,
|
||||
) -> Result<(), std::io::Error> {
|
||||
/*let restore_request = match quick_xml::se::to_string(restore_req) {
|
||||
Ok(buf) => buf,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e));
|
||||
}
|
||||
};*/
|
||||
let restore_request = "".to_string();
|
||||
let restore_request_bytes = restore_request.as_bytes().to_vec();
|
||||
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("restore".to_string(), "".to_string());
|
||||
if version_id != "" {
|
||||
url_values.insert("versionId".to_string(), version_id.to_string());
|
||||
}
|
||||
|
||||
let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes),
|
||||
content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes),
|
||||
content_body: ReaderImpl::Body(restore_request_buffer),
|
||||
content_length: restore_request_bytes.len() as i64,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
if resp_status != http::StatusCode::ACCEPTED && resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -29,10 +29,6 @@ use crate::client::utils::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
pub struct ListAllMyBucketsResult {
|
||||
pub owner: Owner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CommonPrefix {
|
||||
pub prefix: String,
|
||||
@@ -89,6 +85,10 @@ pub struct ListVersionsResult {
|
||||
pub next_version_id_marker: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListBucketResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
contents: Vec<transition_api::ObjectInfo>,
|
||||
@@ -102,6 +102,10 @@ pub struct ListBucketResult {
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListMultipartUploadsResult {
|
||||
bucket: String,
|
||||
key_marker: String,
|
||||
@@ -117,16 +121,15 @@ pub struct ListMultipartUploadsResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct Initiator {
|
||||
id: String,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
pub struct CopyObjectResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectPart {
|
||||
pub etag: String,
|
||||
@@ -260,6 +263,7 @@ pub struct CompletePart {
|
||||
}
|
||||
|
||||
impl CompletePart {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||
match t {
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
@@ -284,11 +288,6 @@ impl CompletePart {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CopyObjectPartResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
#[serde(rename = "CompleteMultipartUpload")]
|
||||
pub struct CompleteMultipartUpload {
|
||||
@@ -357,10 +356,10 @@ impl CompleteMultipartUpload {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CreateBucketConfiguration {
|
||||
pub location: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
||||
)]
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteObject {
|
||||
//api has
|
||||
@@ -368,21 +367,6 @@ pub struct DeleteObject {
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
pub struct DeletedObject {
|
||||
//s3s has
|
||||
pub key: String,
|
||||
pub version_id: String,
|
||||
pub deletemarker: bool,
|
||||
pub deletemarker_version_id: String,
|
||||
}
|
||||
|
||||
pub struct NonDeletedObject {
|
||||
pub key: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteMultiObjects {
|
||||
pub quiet: bool,
|
||||
@@ -402,6 +386,7 @@ impl DeleteMultiObjects {
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WireDeleteObject {
|
||||
@@ -436,8 +421,3 @@ impl DeleteMultiObjects {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DeleteMultiObjectsResult {
|
||||
pub deleted_objects: Vec<DeletedObject>,
|
||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
||||
}
|
||||
|
||||
@@ -27,12 +27,24 @@ use crate::client::utils::base64_decode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
use crate::{disk::DiskAPI, object_api::GetObjectReader};
|
||||
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
||||
// in rustfs-utils' headers module.
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
||||
use s3s::header::{
|
||||
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
|
||||
};
|
||||
|
||||
use enumset::{EnumSet, EnumSetType, enum_set};
|
||||
|
||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
||||
/// this enum is the MinIO-port client's wire vocabulary and stops at the
|
||||
/// standard S3 set (CRC64NVME is its newest member; the RustFS extensions do
|
||||
/// not exist on this client path). The streaming-hash registry lives in
|
||||
/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs) and
|
||||
/// the on-disk xl.meta bitset in `rustfs_rio::ChecksumType`
|
||||
/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an
|
||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
||||
/// each other.
|
||||
#[derive(Debug, EnumSetType, Default)]
|
||||
#[enumset(repr = "u8")]
|
||||
pub enum ChecksumMode {
|
||||
@@ -57,8 +69,6 @@ lazy_static! {
|
||||
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
|
||||
}
|
||||
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
|
||||
|
||||
impl ChecksumMode {
|
||||
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
|
||||
|
||||
@@ -355,6 +365,10 @@ mod tests {
|
||||
pub struct Checksum {
|
||||
checksum_type: ChecksumMode,
|
||||
r: Vec<u8>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
||||
)]
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
|
||||
@@ -32,11 +32,5 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||
|
||||
pub const TOTAL_WORKERS: i64 = 4;
|
||||
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
|
||||
pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;
|
||||
|
||||
@@ -67,6 +67,10 @@ impl<P: Provider + Default> Credentials<P> {
|
||||
Ok(self.creds.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn expire(&mut self) {
|
||||
self.force_refresh = true;
|
||||
}
|
||||
@@ -133,6 +137,10 @@ impl Provider for Static {
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct STSError {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub r#type: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
@@ -141,6 +149,10 @@ pub struct STSError {
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub struct ErrorResponse {
|
||||
pub sts_error: STSError,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub request_id: String,
|
||||
}
|
||||
|
||||
@@ -158,22 +170,3 @@ impl ErrorResponse {
|
||||
return self.sts_error.message.clone();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
match std::str::from_utf8(body) {
|
||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
let body = body_reader.to_vec();
|
||||
let parsed = xml_decoder(&body)?;
|
||||
Ok((body, parsed))
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user