mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 195f19217f |
@@ -66,8 +66,8 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||
@echo "📣 Checking FIPS wording guard..."
|
||||
./scripts/check_fips_wording.sh
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
|
||||
@@ -252,16 +252,10 @@ test-group = 'ecstore-serial-flaky'
|
||||
# cluster, so it keeps the lane's parallel-safe / no-external-dependency
|
||||
# properties. The RustFS warm backend has no loopback guard (that guard is
|
||||
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
|
||||
#
|
||||
# Disk compression (backlog#1848): the `compression` module joins the smoke
|
||||
# lane so the multipart disk-compression roundtrips (restored after
|
||||
# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate.
|
||||
# Single-node servers on random ports with isolated temp dirs — meets the
|
||||
# admission criteria unchanged.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
|
||||
@@ -85,7 +85,7 @@ runs:
|
||||
repo-token: ${{ github.token }}
|
||||
|
||||
- name: Install flatc
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
uses: Nugine/setup-flatc@e7855e994773ce90094a3f1626d4afc9080c23ae # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
|
||||
|
||||
@@ -182,12 +182,7 @@ jobs:
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2
|
||||
# (weekly schedule / manual dispatch only — dormant rio-v2 variant, see
|
||||
# rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md).
|
||||
# The second build below stays despite the reduced cadence: it warms the
|
||||
# rio-v2,e2e-test-hooks feature resolution the scheduled build restores,
|
||||
# which keeps that lane inside its 30-minute timeout.
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
|
||||
@@ -117,9 +117,6 @@ jobs:
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -152,9 +152,6 @@ jobs:
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -536,12 +533,7 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary-rio-v2:
|
||||
name: Build RustFS Debug Binary (rio-v2)
|
||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
||||
# default build, so this full-suite lane runs only on the weekly schedule
|
||||
# and manual dispatch. Per-PR cfg-seam coverage stays with
|
||||
# test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
|
||||
# docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
@@ -832,9 +824,6 @@ jobs:
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
# event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
|
||||
# with it (see the dormant-variant comment on that job).
|
||||
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
|
||||
@@ -94,7 +94,6 @@ jobs:
|
||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -119,7 +118,6 @@ jobs:
|
||||
short_sha=""
|
||||
is_prerelease=false
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
@@ -139,7 +137,6 @@ jobs:
|
||||
# Extract version info from commit message or use commit SHA
|
||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||
source_ref="$HEAD_SHA"
|
||||
|
||||
# Determine build type based on triggering workflow event and ref
|
||||
triggering_event="$TRIGGERING_EVENT"
|
||||
@@ -264,23 +261,6 @@ jobs:
|
||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||
tag_ref="refs/tags/$input_version"
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
if [[ "$input_version" == v* ]]; then
|
||||
tag_ref="refs/tags/${input_version#v}"
|
||||
else
|
||||
tag_ref="refs/tags/v$input_version"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
echo "❌ Release tag not found for Docker build: $input_version"
|
||||
exit 1
|
||||
fi
|
||||
source_ref="$tag_ref"
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
@@ -291,7 +271,6 @@ jobs:
|
||||
echo "short_sha=$short_sha"
|
||||
echo "is_prerelease=$is_prerelease"
|
||||
echo "create_latest=$create_latest"
|
||||
echo "source_ref=$source_ref"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker Build Summary:"
|
||||
@@ -302,7 +281,6 @@ jobs:
|
||||
echo " - Short SHA: $short_sha"
|
||||
echo " - Is prerelease: $is_prerelease"
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
@@ -330,7 +308,6 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
@@ -420,8 +397,7 @@ jobs:
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||
|
||||
@@ -15,35 +15,28 @@
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
||||
# and uploads them to Cloudflare R2.
|
||||
#
|
||||
# Trigger:
|
||||
# - workflow_run: automatically package after "Build and Release" completes
|
||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
||||
# to the GitHub release before packaging starts)
|
||||
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
||||
# - release published: automatically package when a GitHub release is published
|
||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||
#
|
||||
# Flow:
|
||||
# 1. Resolve the triggering Build workflow run for the release tag
|
||||
# 1. Find the Build workflow run for the release tag
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64
|
||||
# 4. Build RPM packages for x86_64 and aarch64
|
||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
||||
# 5. Upload all packages to Cloudflare R2
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
# contents: write is required to upload packages to the GitHub release
|
||||
contents: write
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
on:
|
||||
# Follows the same pattern as docker.yml: run after the release build
|
||||
# workflow completes, so packaging is triggered only by release tags
|
||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
release:
|
||||
types: [ published ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -56,26 +49,13 @@ on:
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
||||
# Tag pushes arrive as event == push with head_branch != main (a
|
||||
# non-main push head_branch is the release tag name). Manual dispatch
|
||||
# stays available as a fallback for backfills and re-runs.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -95,8 +75,8 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
# Determine tag
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
TAG="${HEAD_BRANCH}"
|
||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||
TAG="${{ github.event.release.tag_name }}"
|
||||
elif [[ -n "$INPUT_TAG" ]]; then
|
||||
TAG="$INPUT_TAG"
|
||||
else
|
||||
@@ -113,11 +93,6 @@ jobs:
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Use the Build and Release run that triggered this workflow
|
||||
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
||||
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
# Find the build run that produced this tag
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
@@ -481,54 +456,6 @@ jobs:
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TAG="${{ needs.resolve.outputs.tag }}"
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||
|
||||
# Upload the packages, then refresh the release checksums so the new
|
||||
# assets are covered, matching the binary release flow.
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
||||
gh release upload "$TAG" "$f" --clobber
|
||||
fi
|
||||
done
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
touch "$checksum_file"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
# Remove any stale entry, then append the fresh digest
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
mv "${checksum_file}.tmp" "$checksum_file"
|
||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
||||
gh release upload "$TAG" "$checksum_file" --clobber
|
||||
done
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
+8
-11
@@ -31,7 +31,7 @@ HTTP request
|
||||
→ storage/ecfs (erasure coding, encryption, checksums)
|
||||
→ ecstore (disk pool selection, data distribution)
|
||||
→ rio (reader pipeline: encrypt → compress → hash → write)
|
||||
→ io-core (buffer pool, storage profiling, admission control)
|
||||
→ io-core (zero-copy I/O, buffer pool, direct I/O)
|
||||
→ local disk / remote disk via RPC
|
||||
```
|
||||
|
||||
@@ -55,7 +55,7 @@ rustfs/ # Workspace root (virtual manifest)
|
||||
├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members)
|
||||
│ ├── ecstore/ # Erasure-coded storage engine
|
||||
│ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash)
|
||||
│ ├── io-core/ # Buffer pool, storage profiling, admission control
|
||||
│ ├── io-core/ # Zero-copy I/O, scheduling, buffer pool
|
||||
│ ├── io-metrics/ # I/O metrics collection
|
||||
│ ├── common/ # Shared runtime state, globals, data usage types
|
||||
│ ├── config/ # Configuration types and parsing
|
||||
@@ -101,10 +101,7 @@ refactors.
|
||||
|
||||
The `rustfs` binary crate composes these libraries into the running server.
|
||||
`ecstore` remains the storage engine at the architectural center; its internal
|
||||
module split is tracked under `docs/architecture/`. `rio-v2` is the
|
||||
feature-gated MinIO on-disk format compatibility I/O layer; it ships in no
|
||||
default build (lifecycle:
|
||||
[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)).
|
||||
module split is tracked under `docs/architecture/`.
|
||||
|
||||
## Architecture Invariants
|
||||
|
||||
@@ -134,9 +131,9 @@ default build (lifecycle:
|
||||
why it stays local).
|
||||
- ✅ RESOLVED: `BackpressureConfig` and `DataUsageInfo` each have exactly one
|
||||
definition (`crates/io-core/src/backpressure.rs`,
|
||||
`crates/data-usage/src/data_usage.rs`). The zero-consumer
|
||||
`BackpressureSettings` copy that lingered in io-metrics was removed
|
||||
(rustfs/backlog#1833).
|
||||
`crates/data-usage/src/data_usage.rs`). A zero-consumer
|
||||
`BackpressureSettings` copy lingers in `crates/io-metrics/src/config.rs`;
|
||||
its removal is tracked in rustfs/backlog#1833.
|
||||
|
||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||
storage-level abstractions (objects, buckets, disks, pools).
|
||||
@@ -302,7 +299,7 @@ The binary (`main.rs`) boots in this order:
|
||||
│ │ │
|
||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (core) │ │ (readers) │ │ (buffers) │
|
||||
│ (core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
@@ -314,7 +311,7 @@ The binary (`main.rs`) boots in this order:
|
||||
|
||||
- **"Where does S3 PutObject go?"**
|
||||
`server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
|
||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` supplies buffers
|
||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
|
||||
|
||||
- **"Where are bucket policies enforced?"**
|
||||
`app/bucket_usecase` calls into `crates/policy/`
|
||||
|
||||
Generated
+177
-195
File diff suppressed because it is too large
Load Diff
+63
-64
@@ -41,7 +41,7 @@ members = [
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.1"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -154,7 +154,7 @@ hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.5"
|
||||
http-body-util = "0.1.4"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.2.0" }
|
||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
apache-avro = "0.22.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -182,7 +182,6 @@ quick-xml = "0.41.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
@@ -228,12 +227,12 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.23.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
@@ -284,13 +283,13 @@ rayon = "1.12.0"
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
redis = { version = "1.5.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d358a68783096df1db0c3e314127f2704603b29e" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d7028511a53f69d41ed3c69f36899f9b1aede647" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -313,7 +312,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.24.1" }
|
||||
uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -341,16 +340,16 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.7" }
|
||||
russh = { version = "0.62.6" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -40,7 +40,6 @@ mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
thr = "thr"
|
||||
mis = "mis"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use base64_simd::STANDARD;
|
||||
|
||||
|
||||
@@ -21,13 +21,6 @@ use crate::{
|
||||
Xxhash3, Xxhash64, Xxhash128,
|
||||
};
|
||||
|
||||
// DELIBERATE DUPLICATION of the x-amz-checksum-* names that also exist as
|
||||
// AMZ_CHECKSUM_* in rustfs-utils' headers module (crates/utils/src/http/
|
||||
// headers.rs): this crate is a zero-internal-dependency leaf, so it cannot
|
||||
// import them, and it additionally owns the RustFS extension names
|
||||
// (sha512/xxhash*) that utils does not carry. Values are pinned by the S3
|
||||
// wire protocol; do not merge without a maintainer decision on the leaf
|
||||
// boundary (backlog#1833).
|
||||
pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
|
||||
pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
|
||||
pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
|
||||
@@ -38,10 +31,7 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||
|
||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||
|
||||
@@ -41,14 +41,6 @@ pub const XXHASH_64_NAME: &str = "xxhash64";
|
||||
pub const XXHASH_128_NAME: &str = "xxhash128";
|
||||
pub const MD5_NAME: &str = "md5";
|
||||
|
||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
||||
/// this enum owns the **streaming-hash algorithm registry**, including the
|
||||
/// RustFS extensions (sha512, xxhash3/64/128). The on-disk xl.meta bitset
|
||||
/// lives in `rustfs_rio::ChecksumType` (crates/rio/src/checksum.rs, varint
|
||||
/// bits are append-only), and the MinIO-port client keeps its own
|
||||
/// `ChecksumMode` (crates/ecstore/src/client/checksum.rs). When adding an
|
||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
||||
/// each other.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
#[non_exhaustive]
|
||||
pub enum ChecksumAlgorithm {
|
||||
@@ -476,19 +468,13 @@ impl Checksum for Xxhash64 {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
struct Md5 {
|
||||
hasher: md5::Md5,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
impl Md5 {
|
||||
fn update(&mut self, bytes: &[u8]) {
|
||||
use md5::Digest;
|
||||
|
||||
@@ -42,7 +42,6 @@ chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::last_minute::{self};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub struct ReplicationLatency {
|
||||
// Delays for single and multipart PUT requests
|
||||
upload_histogram: last_minute::LastMinuteHistogram,
|
||||
}
|
||||
|
||||
impl ReplicationLatency {
|
||||
// Merge two ReplicationLatency
|
||||
pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
|
||||
self.upload_histogram.merge(&other.upload_histogram);
|
||||
self
|
||||
}
|
||||
|
||||
// Get upload delay (categorized by object size interval)
|
||||
pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
|
||||
let mut ret = HashMap::new();
|
||||
let avg = self.upload_histogram.get_avg_data();
|
||||
for (i, v) in avg.iter().enumerate() {
|
||||
let avg_duration = v.avg();
|
||||
ret.insert(self.size_tag_to_string(i), avg_duration.as_millis() as u64);
|
||||
}
|
||||
ret
|
||||
}
|
||||
pub fn update(&mut self, size: i64, during: std::time::Duration) {
|
||||
self.upload_histogram.add(size, during);
|
||||
}
|
||||
|
||||
// Simulate the conversion from size tag to string
|
||||
fn size_tag_to_string(&self, tag: usize) -> String {
|
||||
match tag {
|
||||
0 => String::from("Size < 1 KiB"),
|
||||
1 => String::from("Size < 1 MiB"),
|
||||
2 => String::from("Size < 10 MiB"),
|
||||
3 => String::from("Size < 100 MiB"),
|
||||
4 => String::from("Size < 1 GiB"),
|
||||
_ => String::from("Size > 1 GiB"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone, Default)]
|
||||
// pub struct ReplicationLastMinute {
|
||||
// pub last_minute: LastMinuteLatency,
|
||||
// }
|
||||
|
||||
// impl ReplicationLastMinute {
|
||||
// pub fn merge(&mut self, other: ReplicationLastMinute) -> ReplicationLastMinute {
|
||||
// let mut nl = ReplicationLastMinute::default();
|
||||
// nl.last_minute = self.last_minute.merge(&mut other.last_minute);
|
||||
// nl
|
||||
// }
|
||||
|
||||
// pub fn add_size(&mut self, n: i64) {
|
||||
// let t = SystemTime::now()
|
||||
// .duration_since(UNIX_EPOCH)
|
||||
// .expect("Time went backwards")
|
||||
// .as_secs();
|
||||
// self.last_minute.add_all(t - 1, &AccElem { total: t - 1, size: n as u64, n: 1 });
|
||||
// }
|
||||
|
||||
// pub fn get_total(&self) -> AccElem {
|
||||
// self.last_minute.get_total()
|
||||
// }
|
||||
// }
|
||||
|
||||
// impl fmt::Display for ReplicationLastMinute {
|
||||
// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
// let t = self.last_minute.get_total();
|
||||
// write!(f, "ReplicationLastMinute sz= {}, n= {}, dur= {}", t.size, t.n, t.total)
|
||||
// }
|
||||
// }
|
||||
@@ -224,13 +224,6 @@ pub struct HealOpts {
|
||||
pub enum HealAdmissionDropReason {
|
||||
QueueFull,
|
||||
PolicyDropped,
|
||||
/// HS-06: an admin heal start overlaps (same bucket with mutually
|
||||
/// containing prefixes, or the same erasure set) an already running or
|
||||
/// queued task. Only produced when RUSTFS_HEAL_OVERLAP_POLICY=minio_error.
|
||||
AlreadyRunning,
|
||||
/// HS-06: same as [`Self::AlreadyRunning`] but for paths that merely
|
||||
/// contain (or are contained by) the active task's path.
|
||||
OverlappingPaths,
|
||||
}
|
||||
|
||||
impl HealAdmissionDropReason {
|
||||
@@ -238,8 +231,6 @@ impl HealAdmissionDropReason {
|
||||
match self {
|
||||
Self::QueueFull => "queue_full",
|
||||
Self::PolicyDropped => "policy_dropped",
|
||||
Self::AlreadyRunning => "already_running",
|
||||
Self::OverlappingPaths => "overlapping_paths",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -296,9 +287,6 @@ pub enum HealRequestSource {
|
||||
Scanner,
|
||||
AutoHeal,
|
||||
ReadRepair,
|
||||
/// Mission Repair Feed: intents delivered by error paths and replayed
|
||||
/// from the durable MRF journal.
|
||||
Mrf,
|
||||
}
|
||||
|
||||
impl HealRequestSource {
|
||||
@@ -309,7 +297,6 @@ impl HealRequestSource {
|
||||
Self::Scanner => "scanner",
|
||||
Self::AutoHeal => "auto_heal",
|
||||
Self::ReadRepair => "read_repair",
|
||||
Self::Mrf => "mrf",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -326,9 +313,6 @@ pub enum HealChannelCommand {
|
||||
Query {
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
/// Incremental result cursor (HS-06): only items with a sequence
|
||||
/// greater than this are returned; `None` keeps the full snapshot.
|
||||
since_seq: Option<u64>,
|
||||
response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
|
||||
},
|
||||
/// Cancel heal task
|
||||
@@ -534,21 +518,10 @@ async fn receive_heal_channel_response(
|
||||
|
||||
/// Send heal query request
|
||||
pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
|
||||
query_heal_status_since(heal_path, client_token, None).await
|
||||
}
|
||||
|
||||
/// Incremental heal query (HS-06): pass the client's last seen sequence
|
||||
/// number to receive only newer result items.
|
||||
pub async fn query_heal_status_since(
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<HealChannelResponse, String> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
send_heal_command(HealChannelCommand::Query {
|
||||
heal_path,
|
||||
client_token,
|
||||
since_seq,
|
||||
response_tx,
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -572,3 +572,44 @@ mod tests {
|
||||
assert_eq!(total.n, 6);
|
||||
}
|
||||
}
|
||||
|
||||
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LastMinuteHistogram {
|
||||
histogram: Vec<LastMinuteLatency>,
|
||||
size: u32,
|
||||
}
|
||||
|
||||
impl LastMinuteHistogram {
|
||||
pub fn merge(&mut self, other: &LastMinuteHistogram) {
|
||||
for i in 0..self.histogram.len() {
|
||||
self.histogram[i].merge(&other.histogram[i]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(&mut self, size: i64, t: Duration) {
|
||||
let index = size_to_tag(size);
|
||||
self.histogram[index].add(&t);
|
||||
}
|
||||
|
||||
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
|
||||
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
|
||||
for (i, elem) in self.histogram.iter_mut().enumerate() {
|
||||
res[i] = elem.get_total();
|
||||
}
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
fn size_to_tag(size: i64) -> usize {
|
||||
match size {
|
||||
_ if size < 1024 => 0, // sizeLessThan1KiB
|
||||
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
|
||||
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
|
||||
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
|
||||
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
|
||||
_ => 5, // sizeGreaterThan1GiB
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,15 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_stats;
|
||||
// pub mod error;
|
||||
pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
pub mod mrf_channel;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
pub mod trace_bus;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Mission Repair Feed (MRF) intent channel.
|
||||
//!
|
||||
//! Producers on error paths (read decode failure, scanner metadata
|
||||
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
||||
//! heal crate through a global bounded channel. Delivery is strictly
|
||||
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
||||
//! (counting it) when the channel is full or uninitialized — losing one heal
|
||||
//! hint is always preferred over stalling an IO path. Durable replay of
|
||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
||||
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
||||
/// dropping (and counting) intents, never by blocking the producer.
|
||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
||||
|
||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
||||
/// PartialWrite -> Normal).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfKind {
|
||||
/// Erasure decode failed while serving a read (read path).
|
||||
DecodeFailure,
|
||||
/// Scanner classified object metadata as corrupt.
|
||||
MetadataCorruption,
|
||||
/// A write left the object with fewer committed shards than the set size.
|
||||
PartialWrite,
|
||||
}
|
||||
|
||||
impl MrfKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
MrfKind::DecodeFailure => "decode-failure",
|
||||
MrfKind::MetadataCorruption => "metadata-corruption",
|
||||
MrfKind::PartialWrite => "partial-write",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
||||
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
||||
/// intent never re-allocates the strings.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MrfIntent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
/// Version the intent targets, as raw UUID bytes.
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
pub kind: MrfKind,
|
||||
pub enqueued_at_ms: u64,
|
||||
/// Times this intent has already been offered to the heal manager.
|
||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
||||
pub attempts: u8,
|
||||
}
|
||||
|
||||
/// Consumer-side retry ceiling before an intent is given up on.
|
||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
||||
|
||||
impl MrfIntent {
|
||||
/// Rough in-memory footprint used by the queue's byte budget.
|
||||
pub fn estimated_bytes(&self) -> usize {
|
||||
// Struct + strings + version bytes; buckets and objects are usually
|
||||
// far below this bound, so rounding up keeps the budget conservative.
|
||||
64 + self.bucket.len() + self.object.len()
|
||||
}
|
||||
}
|
||||
|
||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
||||
|
||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
||||
/// this before touching the channel so the disabled path stays allocation- and
|
||||
/// sync-free.
|
||||
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
||||
|
||||
/// Override delivery (used at heal-runtime startup from configuration).
|
||||
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
||||
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether producers currently deliver intents.
|
||||
pub fn mrf_delivery_enabled() -> bool {
|
||||
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Create the global MRF channel and return the consumer half. Fails if the
|
||||
/// channel is already initialized (the heal runtime is a singleton).
|
||||
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
||||
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
||||
GLOBAL_MRF_SENDER
|
||||
.set(sender)
|
||||
.map_err(|_| "MRF channel sender already initialized")?;
|
||||
Ok(receiver)
|
||||
}
|
||||
|
||||
/// Best-effort, non-blocking intent delivery from an error path.
|
||||
///
|
||||
/// Returns `true` when the intent was accepted into the channel. `false`
|
||||
/// means the intent was dropped (feature disabled, channel not yet
|
||||
/// initialized, or channel full) — callers must not retry or await; the
|
||||
/// existing read-repair / scanner heal paths remain the safety net.
|
||||
///
|
||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
||||
if !mrf_delivery_enabled() {
|
||||
return false;
|
||||
}
|
||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
||||
return false;
|
||||
};
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
||||
kind,
|
||||
enqueued_at_ms: unix_now_ms(),
|
||||
attempts: 0,
|
||||
};
|
||||
sender.try_send(intent).is_ok()
|
||||
}
|
||||
|
||||
fn unix_now_ms() -> u64 {
|
||||
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
||||
// failure would be a bug rather than something to handle here.
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn intents_estimate_is_conservative() {
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from("bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: Some([0u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
};
|
||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
||||
|
||||
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
||||
let intent = receiver.recv().await.expect("intent should arrive");
|
||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(intent.bucket.as_ref(), "b");
|
||||
|
||||
// Disable delivery: producers become no-ops.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
|
||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
||||
// never blocking.
|
||||
let mut accepted = 0;
|
||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
||||
accepted += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_send_without_channel_is_false() {
|
||||
// This test may run after the tokio test above in the same process;
|
||||
// the singleton semantics make a clean "uninitialized" case hard, so
|
||||
// assert the flag-off behavior only.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
||||
|
||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceKind {
|
||||
Heal,
|
||||
Scanner,
|
||||
}
|
||||
|
||||
impl TraceKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Heal => "heal",
|
||||
Self::Scanner => "scanner",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceFunc {
|
||||
HealTask,
|
||||
HealBucket,
|
||||
HealObject,
|
||||
HealCheckAbandonedParts,
|
||||
HealErasureSetPage,
|
||||
ScannerFolder,
|
||||
ScannerIlmAction,
|
||||
ScannerHealCandidate,
|
||||
Dropped,
|
||||
}
|
||||
|
||||
impl TraceFunc {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HealTask => "heal.Task",
|
||||
Self::HealBucket => "heal.Bucket",
|
||||
Self::HealObject => "heal.Object",
|
||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
||||
Self::ScannerFolder => "scanner.Folder",
|
||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
||||
Self::Dropped => "trace.Dropped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TraceVal {
|
||||
Bool(bool),
|
||||
U64(u64),
|
||||
I64(i64),
|
||||
Str(Arc<str>),
|
||||
}
|
||||
|
||||
impl From<bool> for TraceVal {
|
||||
fn from(value: bool) -> Self {
|
||||
Self::Bool(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for TraceVal {
|
||||
fn from(value: u64) -> Self {
|
||||
Self::U64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for TraceVal {
|
||||
fn from(value: i64) -> Self {
|
||||
Self::I64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for TraceVal {
|
||||
fn from(value: &str) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for TraceVal {
|
||||
fn from(value: String) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceAttr {
|
||||
pub key: &'static str,
|
||||
pub value: TraceVal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceEvent {
|
||||
pub kind: TraceKind,
|
||||
pub func: TraceFunc,
|
||||
pub time: SystemTime,
|
||||
pub bucket: Option<Arc<str>>,
|
||||
pub object: Option<Arc<str>>,
|
||||
pub duration: Duration,
|
||||
pub bytes: u64,
|
||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
||||
}
|
||||
|
||||
impl TraceEvent {
|
||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
func,
|
||||
time: SystemTime::now(),
|
||||
bucket: None,
|
||||
object: None,
|
||||
duration: Duration::ZERO,
|
||||
bytes: 0,
|
||||
attrs: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
||||
self.bucket = Some(bucket.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
||||
self.object = Some(object.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
||||
self.duration = duration;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
||||
self.bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
||||
self.attrs.push(TraceAttr {
|
||||
key,
|
||||
value: value.into(),
|
||||
});
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceBus {
|
||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let capacity = capacity.max(1);
|
||||
let (sender, _receiver) = broadcast::channel(capacity);
|
||||
Self {
|
||||
sender,
|
||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscriber_count(&self) -> usize {
|
||||
self.subscriber_count.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> TraceSubscription {
|
||||
let receiver = self.sender.subscribe();
|
||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
||||
TraceSubscription {
|
||||
receiver,
|
||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
if self.subscriber_count() == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.sender.send(Arc::new(build())).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TraceBus {
|
||||
fn default() -> Self {
|
||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceSubscription {
|
||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceSubscription {
|
||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
|
||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
||||
self.receiver.try_recv()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceSubscription {
|
||||
fn drop(&mut self) {
|
||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn global_trace_bus() -> &'static TraceBus {
|
||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
||||
}
|
||||
|
||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
||||
global_trace_bus().subscribe()
|
||||
}
|
||||
|
||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
global_trace_bus().emit(build)
|
||||
}
|
||||
|
||||
pub fn trace_subscriber_count() -> usize {
|
||||
global_trace_bus().subscriber_count()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
#[test]
|
||||
fn trace_emit_skips_builder_without_subscribers() {
|
||||
let bus = TraceBus::new(4);
|
||||
let built = AtomicUsize::new(0);
|
||||
|
||||
let sent = bus.emit(|| {
|
||||
built.fetch_add(1, Ordering::Relaxed);
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
});
|
||||
|
||||
assert!(!sent);
|
||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trace_subscriber_receives_event() {
|
||||
let bus = TraceBus::new(4);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
assert!(bus.emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("bucket")
|
||||
.with_object("object")
|
||||
.with_duration(Duration::from_millis(7))
|
||||
.with_bytes(11)
|
||||
.with_attr("dry", true)
|
||||
}));
|
||||
|
||||
let event = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect("subscriber should receive emitted trace event");
|
||||
|
||||
assert_eq!(event.kind, TraceKind::Heal);
|
||||
assert_eq!(event.func, TraceFunc::HealObject);
|
||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(event.object.as_deref(), Some("object"));
|
||||
assert_eq!(event.duration, Duration::from_millis(7));
|
||||
assert_eq!(event.bytes, 11);
|
||||
assert_eq!(
|
||||
event.attrs.as_slice(),
|
||||
&[TraceAttr {
|
||||
key: "dry",
|
||||
value: TraceVal::Bool(true)
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_subscription_drop_decrements_count() {
|
||||
let bus = TraceBus::new(4);
|
||||
let subscription = bus.subscribe();
|
||||
|
||||
assert_eq!(bus.subscriber_count(), 1);
|
||||
drop(subscription);
|
||||
assert_eq!(bus.subscriber_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
||||
let bus = TraceBus::new(2);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
for index in 0_u64..4 {
|
||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
||||
}
|
||||
|
||||
let err = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,9 @@
|
||||
|
||||
//! Shared backpressure policy type.
|
||||
//!
|
||||
//! This module only carries the watermark policy; the admission primitive it
|
||||
//! projects into lives in `rustfs-io-core`.
|
||||
//! The runtime backpressure implementation (byte-watermark pipes and
|
||||
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
||||
//! carries the watermark policy type that implementation shares.
|
||||
|
||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||
|
||||
|
||||
@@ -353,11 +353,6 @@ pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
|
||||
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
|
||||
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
||||
|
||||
/// Default detailed PUT stage metrics enabled
|
||||
/// Default value: false
|
||||
/// Environment variable: RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED
|
||||
pub const DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED: bool = false;
|
||||
|
||||
/// Default logs export enabled
|
||||
/// It is used to enable or disable exporting logs
|
||||
/// Default value: true
|
||||
|
||||
@@ -177,40 +177,3 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
||||
|
||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||
|
||||
/// Environment variable that toggles the MRF (mission repair feed) intent
|
||||
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
||||
/// unconsumed intents are replayed from the durable journal after a restart.
|
||||
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
||||
|
||||
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
||||
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
||||
|
||||
/// Environment variable for the MRF journal byte budget. The journal is
|
||||
/// compacted once its on-disk size crosses this bound.
|
||||
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
||||
|
||||
/// Environment variable for the MRF journal replay batch size (intents per
|
||||
/// replay push round).
|
||||
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
||||
|
||||
/// Default behavior keeps the MRF intent pipeline enabled.
|
||||
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
||||
|
||||
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
||||
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
||||
|
||||
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
||||
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Default MRF replay batch size.
|
||||
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
||||
|
||||
/// Environment variable selecting how admin heal starts behave when the
|
||||
/// requested path overlaps an already running or queued heal: `merge`
|
||||
/// (default, keep today's dedup/merge semantics) or `minio_error` (return a
|
||||
/// typed already-running / overlapping-paths rejection like madmin).
|
||||
pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY";
|
||||
|
||||
/// Default overlap policy: merge duplicate/overlapping requests.
|
||||
pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge";
|
||||
|
||||
@@ -137,37 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
/// Request the object-transaction fencing contract used by storage-owned
|
||||
/// cleanup receipts and lock-window optimizations.
|
||||
///
|
||||
/// This is fail-closed: enabling the writer without a live fleet proof rejects
|
||||
/// the commit rather than silently using a legacy-safe path.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_WRITE: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// object transaction fencing contract.
|
||||
pub const ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE);
|
||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED);
|
||||
|
||||
/// Request preserving legacy per-part checksum metadata during data movement.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested confirmation that every serving node understands the
|
||||
/// data-movement per-part checksum sidecar.
|
||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
|
||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -234,31 +203,6 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||
|
||||
/// Enable foreground PutObject request admission.
|
||||
///
|
||||
/// This is an experimental, default-off foreground write backpressure gate for
|
||||
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
||||
/// path and only the existing request counters are updated.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
||||
|
||||
/// Maximum foreground PutObject requests admitted concurrently per process.
|
||||
///
|
||||
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
||||
/// A value of `0` disables the gate even when the enable flag is present, so a
|
||||
/// partially configured rollout cannot reject every PUT.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
||||
|
||||
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
||||
///
|
||||
/// Once this timeout expires the request fails before body ingest/storage
|
||||
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
||||
|
||||
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
||||
|
||||
/// Environment variable for minimum GetObject timeout in seconds.
|
||||
///
|
||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||
@@ -705,22 +649,4 @@ mod remote_version_state_tests {
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED,
|
||||
"RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_transaction_fencing_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_OBJECT_TRANSACTION_FENCING_WRITE, "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED,
|
||||
"RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,8 +81,7 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
|
||||
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
||||
/// Runtime env var controlling the transition worker count.
|
||||
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
||||
/// Runtime env var controlling the ILM expiry worker count. A set, parsable,
|
||||
/// non-zero value wins; anything else falls back to `min(cpus, 16)`.
|
||||
/// Runtime env var controlling the expiry worker count.
|
||||
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
|
||||
/// Runtime env var controlling the absolute maximum transition workers.
|
||||
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
||||
|
||||
@@ -44,10 +44,6 @@ pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENAB
|
||||
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
||||
|
||||
/// Enables detailed per-stage PUT metrics. Disabled by default because each
|
||||
/// PUT records multiple timers and histograms when attribution is active.
|
||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
@@ -145,7 +141,6 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -870,157 +870,6 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
||||
/// scanner's writer-side cache has the same map type).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or is
|
||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
||||
/// was cut (largest first).
|
||||
pub fn prefix_usage_in_cache(
|
||||
cache: &HashMap<String, DataUsageEntry>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> Option<PrefixUsageQuery> {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
let root = if prefix.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
let child_prefix = format!("{root}/");
|
||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
||||
.children
|
||||
.iter()
|
||||
.filter_map(|child_key| {
|
||||
let child = cache.get(child_key)?;
|
||||
let child_flat = flatten_entry(cache, child, 1)?;
|
||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
||||
// slash marks a directory object and is display-only here.
|
||||
let name = child_key
|
||||
.strip_prefix(child_prefix.as_str())
|
||||
.unwrap_or(child_key.as_str())
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
Some(PrefixUsageEntry {
|
||||
prefix: name,
|
||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
sub_prefixes.sort_by(|left, right| {
|
||||
right
|
||||
.usage
|
||||
.size
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
let truncated = sub_prefixes.len() > max_entries;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
|
||||
Some(PrefixUsageQuery {
|
||||
usage,
|
||||
compacted: entry.compacted,
|
||||
truncated,
|
||||
sub_prefixes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
||||
|
||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
||||
return None;
|
||||
}
|
||||
let mut flattened = DataUsageEntry::default();
|
||||
if !flattened.checked_merge(root) {
|
||||
return None;
|
||||
}
|
||||
flattened.compacted = root.compacted;
|
||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
||||
// child edge pointing back at the root's own key is still terminated by
|
||||
// the visited set on first encounter.
|
||||
let mut visited: HashSet<&str> = HashSet::new();
|
||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
||||
while let Some((key, child_depth)) = pending.pop() {
|
||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let entry = cache.get(key)?;
|
||||
if !flattened.checked_merge(entry) {
|
||||
return None;
|
||||
}
|
||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
||||
}
|
||||
flattened.children.clear();
|
||||
Some(flattened)
|
||||
}
|
||||
|
||||
/// Flattened counters of one prefix subtree, as returned by
|
||||
/// [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageSummary {
|
||||
pub size: u64,
|
||||
pub objects: u64,
|
||||
pub versions: u64,
|
||||
pub delete_markers: u64,
|
||||
}
|
||||
|
||||
impl PrefixUsageSummary {
|
||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
||||
Self {
|
||||
size: entry.size as u64,
|
||||
objects: entry.objects as u64,
|
||||
versions: entry.versions as u64,
|
||||
delete_markers: entry.delete_markers as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add another set's counters into this one (entries are partitioned by
|
||||
/// set, so per-set results sum).
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
self.size = self.size.saturating_add(other.size);
|
||||
self.objects = self.objects.saturating_add(other.objects);
|
||||
self.versions = self.versions.saturating_add(other.versions);
|
||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||
}
|
||||
}
|
||||
|
||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct PrefixUsageEntry {
|
||||
pub prefix: String,
|
||||
pub usage: PrefixUsageSummary,
|
||||
}
|
||||
|
||||
/// Result of [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageQuery {
|
||||
pub usage: PrefixUsageSummary,
|
||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
||||
/// but no sub-prefix breakdown exists on disk.
|
||||
pub compacted: bool,
|
||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
||||
pub truncated: bool,
|
||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||
}
|
||||
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
@@ -1148,21 +997,6 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
||||
/// full counters instead of first-level sizes only).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or
|
||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
||||
/// which the `compacted` flag reports so callers can say why the
|
||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
||||
/// `max_entries` and was cut (largest first).
|
||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||
}
|
||||
|
||||
pub fn force_compact(&mut self, limit: usize) {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
@@ -2064,126 +1898,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
||||
/// distinct counters so aggregation is observable.
|
||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
||||
cache.replace(
|
||||
path,
|
||||
parent,
|
||||
DataUsageEntry {
|
||||
size,
|
||||
objects,
|
||||
versions,
|
||||
delete_markers,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
insert("bucket", "", 0, 0, 0, 0);
|
||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
||||
cache
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let root = cache
|
||||
.prefix_usage("bucket", "", 100)
|
||||
.expect("root query must find the bucket entry");
|
||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
||||
assert_eq!(root.usage.objects, 5);
|
||||
assert_eq!(root.usage.versions, 7);
|
||||
assert_eq!(root.usage.delete_markers, 2);
|
||||
assert!(!root.compacted);
|
||||
assert!(!root.truncated);
|
||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
||||
// each flattened to its own subtree total.
|
||||
let names: Vec<(&str, u64)> = root
|
||||
.sub_prefixes
|
||||
.iter()
|
||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
||||
.collect();
|
||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
||||
assert_eq!(b.usage.size, 240);
|
||||
assert_eq!(b.usage.versions, 5);
|
||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["c", "d"]);
|
||||
|
||||
// Prefix slashes are normalized away.
|
||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
||||
assert_eq!(slashed.usage.size, 240);
|
||||
|
||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_reports_and_respects_truncation() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_marks_compacted_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 999,
|
||||
objects: 9,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
||||
assert_eq!(compacted.usage.size, 999);
|
||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
||||
// whole query, not unbounded recursion.
|
||||
let mut cache = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b".to_string());
|
||||
}
|
||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
||||
// The unaffected sibling still answers.
|
||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
||||
|
||||
// A child key with no entry (dangling link) is rejected rather than
|
||||
// silently dropped: half a tree would under-report usage.
|
||||
let mut dangling = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b/ghost".to_string());
|
||||
}
|
||||
assert!(
|
||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
||||
"a dangling child link must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
@@ -32,7 +32,6 @@ use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -52,11 +51,6 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
const TEST_PORT_RANGE: u16 = 40_000;
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
@@ -73,68 +67,7 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
struct PortAllocatorGuard;
|
||||
|
||||
impl PortAllocatorGuard {
|
||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
loop {
|
||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
remove_stale_port_allocator_lock();
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PortAllocatorGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port() -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
||||
return;
|
||||
};
|
||||
let Ok(modified) = metadata.modified() else {
|
||||
return;
|
||||
};
|
||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(log_path) = log_path else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -572,21 +505,10 @@ impl RustFSTestEnvironment {
|
||||
/// Find an available port for the test
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let mut next_port = read_next_test_port();
|
||||
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
drop(listener);
|
||||
return Ok(port);
|
||||
}
|
||||
}
|
||||
|
||||
Err("no available E2E test port found".into())
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let port = listener.local_addr()?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
/// Kill any existing RustFS processes
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
@@ -26,15 +25,6 @@ fn generate_compressible_data(size: usize) -> Vec<u8> {
|
||||
data
|
||||
}
|
||||
|
||||
/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part
|
||||
/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload
|
||||
/// Pending truncation (rustfs/rustfs#5957).
|
||||
fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
|
||||
(0..size)
|
||||
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
|
||||
let bucket_path = PathBuf::from(temp_dir).join(bucket);
|
||||
let mut part_files = Vec::new();
|
||||
@@ -65,14 +55,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
// Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR
|
||||
// capture (survives the temp-dir cleanup on Drop and is uploaded as a CI
|
||||
// artifact); without the env var the child inherits stdio as before.
|
||||
let mut command = Command::new(&binary_path);
|
||||
command
|
||||
let process = Command::new(&binary_path)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
@@ -81,9 +66,8 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
]);
|
||||
crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?;
|
||||
let process = command.spawn()?;
|
||||
])
|
||||
.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
@@ -170,647 +154,3 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MULTIPART_COMPRESSION_BUCKET: &str = "compression-multipart-bucket";
|
||||
const MPU_PART1_SIZE: usize = 5 * 1024 * 1024;
|
||||
const MPU_PART2_SIZE: usize = 1024 * 1024;
|
||||
|
||||
async fn multipart_upload(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
parts: &[&[u8]],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::with_capacity(parts.len());
|
||||
for (i, part) in parts.iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.to_vec()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_range(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
range: &str,
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let response = client.get_object().bucket(bucket).key(key).range(range).send().await?;
|
||||
Ok(response.body.collect().await?.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// Multipart disk compression roundtrip: parts are written as independent
|
||||
/// compressed streams and every GET shape must reassemble the original bytes
|
||||
/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression
|
||||
/// entirely).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)");
|
||||
|
||||
// Full GET must reassemble both independently compressed parts.
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range fully inside part 1.
|
||||
let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?;
|
||||
assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// Range fully inside part 2.
|
||||
let part2_start = MPU_PART1_SIZE + 4096;
|
||||
let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1;
|
||||
let range_inside_part2 = fetch_range(
|
||||
&client,
|
||||
MULTIPART_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={part2_start}-{part2_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_inside_part2[..],
|
||||
&original_data[part2_start..part2_end + 1],
|
||||
"part-2 range mismatch"
|
||||
);
|
||||
|
||||
// Suffix range (last 128 KiB, entirely in part 2).
|
||||
let suffix_len = 128 * 1024;
|
||||
let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
// partNumber GETs must return each original part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
info!("Multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket";
|
||||
|
||||
/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the
|
||||
/// disk-compression path runs and each part is stored as many compressed blocks — the shape that
|
||||
/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return
|
||||
/// the exact original bytes, and the stored size must show the data really was compressed.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart high-ratio binary compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-high-ratio.txt";
|
||||
let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7);
|
||||
let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
|
||||
// margin still proves the parts were stored compressed rather than raw or double-encoded.
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size as u64) / 2,
|
||||
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
|
||||
);
|
||||
info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)");
|
||||
|
||||
info!("step: full GET");
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary.
|
||||
info!("step: boundary range GET");
|
||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_HIGH_RATIO_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
info!("step: partNumber GET");
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch");
|
||||
|
||||
info!("Multipart high-ratio binary compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket";
|
||||
const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024;
|
||||
const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied
|
||||
/// range must be decompressed on read and re-compressed into the destination part, so the final
|
||||
/// object has to match "source prefix + uploaded tail" byte for byte.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting multipart upload-part-copy compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
|
||||
// Source object: a plain PUT that goes through the single-stream compression path.
|
||||
let source_key = "copy-source.txt";
|
||||
let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(source_key)
|
||||
.body(ByteStream::from(source_data.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Destination object: part 1 copied from the source, part 2 uploaded directly.
|
||||
let target_key = "copy-target.txt";
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec();
|
||||
expected_data.extend_from_slice(&part2);
|
||||
let total_size = expected_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let copy_part = client
|
||||
.upload_part_copy()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}"))
|
||||
.copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1))
|
||||
.send()
|
||||
.await?;
|
||||
let copy_etag = copy_part
|
||||
.copy_part_result()
|
||||
.and_then(|r| r.e_tag())
|
||||
.ok_or("missing copy part etag")?
|
||||
.to_string();
|
||||
|
||||
let uploaded_part = client
|
||||
.upload_part()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build())
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_part.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
||||
.key(target_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch");
|
||||
|
||||
info!("Multipart upload-part-copy compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket";
|
||||
const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024;
|
||||
|
||||
/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one
|
||||
/// compressed part stream, and a suffix range must resolve inside the trailing part.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Starting three-part multipart compression partNumber test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-three-parts.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
original_data.extend_from_slice(&part3);
|
||||
let total_size = original_data.len();
|
||||
|
||||
multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
||||
);
|
||||
|
||||
// Every partNumber GET must return exactly the bytes of the corresponding uploaded part.
|
||||
for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] {
|
||||
let response = client
|
||||
.get_object()
|
||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(part_number)
|
||||
.send()
|
||||
.await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
||||
}
|
||||
|
||||
// Suffix range (last 64 KiB) resolves inside the trailing part.
|
||||
let suffix_len = 64 * 1024;
|
||||
let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
||||
|
||||
info!("Three-part multipart compression partNumber test passed");
|
||||
env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
|
||||
|
||||
async fn start_rustfs_with_compression_and_sse(
|
||||
env: &mut RustFSTestEnvironment,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use base64::Engine;
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
// Server output goes to a file inside the per-test temp dir so a failing
|
||||
// run can be diagnosed from the child's logs.
|
||||
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
|
||||
let server_log_err = server_log.try_clone()?;
|
||||
let process = Command::new(&binary_path)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
|
||||
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
|
||||
.stdout(std::process::Stdio::from(server_log))
|
||||
.stderr(std::process::Stdio::from(server_log_err))
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
"--access-key",
|
||||
&env.access_key,
|
||||
"--secret-key",
|
||||
&env.secret_key,
|
||||
&env.temp_dir,
|
||||
])
|
||||
.spawn()?;
|
||||
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
||||
/// shape must still return the original plaintext bytes. Physical size must shrink because the
|
||||
/// compression runs before encryption.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
|
||||
init_logging();
|
||||
info!("Starting SSE-S3 multipart compression roundtrip test");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_with_compression_and_sse(&mut env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
|
||||
let object_key = "multipart-sse-compressible.txt";
|
||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
||||
let mut original_data = part1.clone();
|
||||
original_data.extend_from_slice(&part2);
|
||||
let total_size = original_data.len();
|
||||
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed_parts = Vec::new();
|
||||
for (i, part) in [&part1, &part2].into_iter().enumerate() {
|
||||
let part_number = (i + 1) as i32;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(upload.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head_response = client
|
||||
.head_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
head_response.content_length().unwrap_or(0) as usize,
|
||||
total_size,
|
||||
"Content-Length should be the logical object size"
|
||||
);
|
||||
assert_eq!(
|
||||
head_response.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::Aes256),
|
||||
"HEAD must report SSE-S3"
|
||||
);
|
||||
|
||||
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
|
||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
||||
assert!(
|
||||
total_physical_size < (total_size / 2) as u64,
|
||||
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
|
||||
);
|
||||
|
||||
let get_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(downloaded.len(), total_size);
|
||||
assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch");
|
||||
|
||||
// Range crossing the part boundary must decrypt and decompress across parts.
|
||||
let boundary_start = MPU_PART1_SIZE - 64 * 1024;
|
||||
let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1;
|
||||
let range_crossing = fetch_range(
|
||||
&client,
|
||||
MPU_SSE_COMPRESSION_BUCKET,
|
||||
object_key,
|
||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
&range_crossing[..],
|
||||
&original_data[boundary_start..boundary_end + 1],
|
||||
"SSE-S3 boundary-crossing range mismatch"
|
||||
);
|
||||
|
||||
// partNumber GET for the trailing part.
|
||||
let part2_response = client
|
||||
.get_object()
|
||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
||||
.key(object_key)
|
||||
.part_number(2)
|
||||
.send()
|
||||
.await?;
|
||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
||||
assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch");
|
||||
|
||||
info!("SSE-S3 multipart compression roundtrip test passed");
|
||||
env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end
|
||||
|
||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions.
|
||||
|
||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||
|
||||
@@ -30,12 +30,10 @@ use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput,
|
||||
DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput,
|
||||
GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||
PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -78,25 +76,6 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
];
|
||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
];
|
||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
/// Wire prefix of the SSE-C passthrough replication transport headers
|
||||
/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a
|
||||
/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with
|
||||
/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO /
|
||||
/// generic S3, which silently discard unknown x-* headers.
|
||||
const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-";
|
||||
const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm";
|
||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||
|
||||
@@ -112,9 +91,6 @@ pub enum Operation {
|
||||
GetObject,
|
||||
HeadObject,
|
||||
DeleteObject,
|
||||
GetObjectTagging,
|
||||
PutObjectTagging,
|
||||
DeleteObjectTagging,
|
||||
ListObjectVersions,
|
||||
CreateMultipartUpload,
|
||||
UploadPart,
|
||||
@@ -142,61 +118,6 @@ pub enum FaultAction {
|
||||
WrongEtag,
|
||||
}
|
||||
|
||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||
/// sender-side tests can assert what a real target would receive.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReplicationTimestampHeaders {
|
||||
pub tagging: Option<String>,
|
||||
pub retention: Option<String>,
|
||||
pub legalhold: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicationTimestampHeaders {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-proxy related headers observed on a request, journaled so proxy
|
||||
/// tests can assert the exact wire contract: the anti-loop marker present,
|
||||
/// the replication-check exemption absent, and the client SSE-C key family
|
||||
/// forwarded verbatim. The SSE-C key value itself is never retained — only
|
||||
/// its presence.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProxyHeaderSnapshot {
|
||||
pub source_proxy_request: Option<String>,
|
||||
pub replication_check: Option<String>,
|
||||
pub ssec_algorithm: Option<String>,
|
||||
pub ssec_key_present: bool,
|
||||
pub ssec_key_md5: Option<String>,
|
||||
/// Whether the request carried any `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header, so fail-closed tests can assert the
|
||||
/// sender really shipped the material a dropping target discarded.
|
||||
pub ssec_transport_present: bool,
|
||||
}
|
||||
|
||||
impl ProxyHeaderSnapshot {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"])
|
||||
.map(bounded_journal_value),
|
||||
replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"),
|
||||
ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value),
|
||||
ssec_transport_present: headers
|
||||
.keys()
|
||||
.any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Credential-free request metadata retained for deterministic assertions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestRecord {
|
||||
@@ -210,8 +131,6 @@ pub struct RequestRecord {
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub proxy_headers: ProxyHeaderSnapshot,
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
@@ -227,10 +146,6 @@ struct ControlState {
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
assign_own_multipart_version_ids: bool,
|
||||
/// MinIO-like mode: silently discard non-whitelisted replication
|
||||
/// transport headers instead of storing them (see
|
||||
/// [`REPLICATION_SSE_TRANSPORT_PREFIX`]).
|
||||
drop_unlisted_replication_headers: bool,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
total_bytes: usize,
|
||||
@@ -252,12 +167,6 @@ struct ObjectVersion {
|
||||
delete_marker: bool,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
/// Object tags as ordered key/value pairs (PutObjectTagging replaces the
|
||||
/// whole set, DeleteObjectTagging clears it).
|
||||
tags: Vec<(String, String)>,
|
||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||
/// target behavior); empty when the drop mode discarded them.
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -267,7 +176,6 @@ struct MultipartState {
|
||||
version_id: String,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
parts: BTreeMap<i32, MultipartPart>,
|
||||
}
|
||||
|
||||
@@ -488,15 +396,6 @@ impl FakeS3Target {
|
||||
|
||||
/// Mint own version ids for the multipart path only — models a target
|
||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||
/// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header instead of storing it. The default (off)
|
||||
/// models a RustFS target, which preserves the headers and echoes SSE-C
|
||||
/// evidence (`x-amz-server-side-encryption-customer-algorithm`) on
|
||||
/// HEAD/GET of the replica.
|
||||
pub fn drop_unlisted_replication_headers(&self, enabled: bool) {
|
||||
lock(&self.backend.store).drop_unlisted_replication_headers = enabled;
|
||||
}
|
||||
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
@@ -637,17 +536,7 @@ impl S3Access for FaultAccess {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok());
|
||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||
let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers());
|
||||
let fault = record_request(
|
||||
&self.control,
|
||||
operation,
|
||||
context.method().clone(),
|
||||
parsed,
|
||||
content_length,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
);
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||
if let Some(RequestFault {
|
||||
action: FaultAction::Status(status),
|
||||
..
|
||||
@@ -686,9 +575,6 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
||||
"GetObject" => Operation::GetObject,
|
||||
"HeadObject" => Operation::HeadObject,
|
||||
"DeleteObject" => Operation::DeleteObject,
|
||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||
"UploadPart" => Operation::UploadPart,
|
||||
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
||||
@@ -703,8 +589,6 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
content_length: Option<u64>,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
proxy_headers: ProxyHeaderSnapshot,
|
||||
) -> Option<RequestFault> {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
@@ -729,8 +613,6 @@ fn record_request(
|
||||
part_number: parsed.part_number,
|
||||
content_length,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
@@ -797,15 +679,6 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||
(&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::GetObjectTagging
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::PutObjectTagging
|
||||
}
|
||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::DeleteObjectTagging
|
||||
}
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
@@ -873,29 +746,6 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
||||
Ok(version_id.to_string())
|
||||
}
|
||||
|
||||
/// Capture the SSE-C passthrough transport headers a replication PUT carried.
|
||||
/// Returns an empty set in the MinIO-like drop mode.
|
||||
fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> {
|
||||
if drop_unlisted {
|
||||
return Vec::new();
|
||||
}
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX))
|
||||
.filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// SSE-C evidence a RustFS-like target echoes for a stored passthrough
|
||||
/// replica: the customer algorithm restored from the transport headers.
|
||||
fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option<String> {
|
||||
version
|
||||
.replication_sse_headers
|
||||
.iter()
|
||||
.find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER)
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
|
||||
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
||||
header_value(headers, &SOURCE_ETAG_HEADERS)
|
||||
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
||||
@@ -1243,33 +1093,6 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option<
|
||||
Ok(version.clone())
|
||||
}
|
||||
|
||||
/// Replace (or clear, with an empty vec) the tag set of the addressed
|
||||
/// version, returning its version id. Mirrors `find_version` addressing:
|
||||
/// explicit version id or the latest version, delete markers rejected.
|
||||
fn set_version_tags(
|
||||
state: &mut StoreState,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
tags: Vec<(String, String)>,
|
||||
) -> S3Result<String> {
|
||||
// Resolve first (immutable) so the error paths match find_version.
|
||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||
let versions = state
|
||||
.buckets
|
||||
.get_mut(bucket)
|
||||
.expect("bucket existence checked by find_version")
|
||||
.objects
|
||||
.get_mut(key)
|
||||
.expect("key existence checked by find_version");
|
||||
let version = versions
|
||||
.iter_mut()
|
||||
.find(|version| version.version_id == resolved)
|
||||
.expect("version existence checked by find_version");
|
||||
version.tags = tags;
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl S3 for FakeBackend {
|
||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
@@ -1366,10 +1189,7 @@ impl S3 for FakeBackend {
|
||||
let input = req.input;
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let (assign_own, drop_unlisted) = {
|
||||
let state = lock(&self.store);
|
||||
(state.assign_own_version_ids, state.drop_unlisted_replication_headers)
|
||||
};
|
||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||
let version_id = new_version_id(&headers, assign_own)?;
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
Some(value) => value,
|
||||
@@ -1386,8 +1206,6 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
};
|
||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1408,7 +1226,6 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectOutput {
|
||||
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
||||
@@ -1418,7 +1235,6 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
@@ -1433,7 +1249,6 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(HeadObjectOutput {
|
||||
content_length: Some(version.body.len() as i64),
|
||||
@@ -1442,79 +1257,12 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let tag_set: TagSet = version
|
||||
.tags
|
||||
.into_iter()
|
||||
.map(|(key, value)| Tag {
|
||||
key: Some(key),
|
||||
value: Some(value),
|
||||
})
|
||||
.collect();
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectTaggingOutput {
|
||||
tag_set,
|
||||
version_id: Some(ObjectVersionId::from(version.version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let tags = input
|
||||
.tagging
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default()))
|
||||
.collect();
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(DeleteObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
@@ -1591,8 +1339,6 @@ impl S3 for FakeBackend {
|
||||
delete_marker: true,
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1620,10 +1366,9 @@ impl S3 for FakeBackend {
|
||||
ensure_upload_budget(&state)?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let upload_id = Uuid::new_v4().to_string();
|
||||
// Read the flags before the mutable borrow of `state.uploads` below
|
||||
// Read the flag before the mutable borrow of `state.uploads` below
|
||||
// (and never re-lock the store: the mutex is not reentrant).
|
||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||
let drop_unlisted = state.drop_unlisted_replication_headers;
|
||||
let version_id = new_version_id(&headers, mint_own)?;
|
||||
state.uploads.insert(
|
||||
upload_id.clone(),
|
||||
@@ -1633,7 +1378,6 @@ impl S3 for FakeBackend {
|
||||
version_id,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -1771,7 +1515,6 @@ impl S3 for FakeBackend {
|
||||
version_id: upload.version_id.clone(),
|
||||
content_type: upload.content_type.clone(),
|
||||
metadata: upload.metadata.clone(),
|
||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
selected,
|
||||
@@ -1798,8 +1541,6 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: upload.content_type,
|
||||
metadata: upload.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: upload.replication_sse_headers,
|
||||
};
|
||||
let mut state = lock(&self.store);
|
||||
let current = state
|
||||
@@ -1958,111 +1699,6 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("plain")
|
||||
.body(ByteStream::from_static(b"plain"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("stamped")
|
||||
.body(ByteStream::from_static(b"stamped"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let requests = target.requests();
|
||||
let plain = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||
.expect("plain PUT must be journaled");
|
||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||
|
||||
let stamped = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||
.expect("stamped PUT must be journaled");
|
||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Default mode is RustFS-like: SSE-C passthrough transport headers are
|
||||
/// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is
|
||||
/// MinIO-like: the headers are silently discarded, so no evidence comes
|
||||
/// back — the exact difference the N2 fail-closed audit keys on. Both
|
||||
/// modes journal that the sender shipped the transport headers.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
let put_with_transport_headers = |key: &'static str| {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"ciphertext"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-replication-ssec-algorithm", "AES256");
|
||||
headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA==");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
};
|
||||
|
||||
put_with_transport_headers("kept").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), Some("AES256"));
|
||||
let get = client.get_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(get.sse_customer_algorithm(), Some("AES256"));
|
||||
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
put_with_transport_headers("dropped").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("dropped").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence");
|
||||
|
||||
let requests = target.requests();
|
||||
for key in ["kept", "dropped"] {
|
||||
let record = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key))
|
||||
.expect("PUT must be journaled");
|
||||
assert!(
|
||||
record.proxy_headers.ssec_transport_present,
|
||||
"the journal must prove the sender shipped the transport headers for {key}"
|
||||
);
|
||||
}
|
||||
let plain_head = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::HeadObject)
|
||||
.expect("HEAD must be journaled");
|
||||
assert!(!plain_head.proxy_headers.ssec_transport_present);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! assert_sdk_error {
|
||||
($error:expr, $status:expr, $code:expr) => {{
|
||||
let error = &$error;
|
||||
@@ -3328,7 +2964,6 @@ mod tests {
|
||||
version_id: index.to_string(),
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
replication_sse_headers: Vec::new(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -3350,8 +2985,6 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
Some(0),
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
}
|
||||
let records = lock(&control).requests.clone();
|
||||
@@ -3373,8 +3006,6 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
None,
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
|
||||
@@ -67,9 +67,6 @@ type MetricValues = Arc<Mutex<BTreeMap<String, MetricPointVersions>>>;
|
||||
|
||||
const KIB: usize = 1024;
|
||||
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
||||
/// Physical bytes the erasure layer pulled from disk, emitted per shard read by
|
||||
/// `crates/ecstore/src/erasure/coding/decode.rs`.
|
||||
const SHARD_READ_BYTES_COUNTER: &str = "rustfs_io_get_object_shard_read_observed_bytes_total";
|
||||
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
@@ -149,7 +146,6 @@ struct OtlpMetricCollector {
|
||||
decode_values: MetricValues,
|
||||
fallback_values: MetricValues,
|
||||
decode_error_values: MetricValues,
|
||||
shard_read_values: MetricValues,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
@@ -161,12 +157,10 @@ impl OtlpMetricCollector {
|
||||
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let shard_read_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let task_values = values.clone();
|
||||
let task_decode_values = decode_values.clone();
|
||||
let task_fallback_values = fallback_values.clone();
|
||||
let task_decode_error_values = decode_error_values.clone();
|
||||
let task_shard_read_values = shard_read_values.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
@@ -176,7 +170,6 @@ impl OtlpMetricCollector {
|
||||
let decode_values = task_decode_values.clone();
|
||||
let fallback_values = task_fallback_values.clone();
|
||||
let decode_error_values = task_decode_error_values.clone();
|
||||
let shard_read_values = task_shard_read_values.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(
|
||||
@@ -188,7 +181,6 @@ impl OtlpMetricCollector {
|
||||
decode_values.clone(),
|
||||
fallback_values.clone(),
|
||||
decode_error_values.clone(),
|
||||
shard_read_values.clone(),
|
||||
)
|
||||
}),
|
||||
)
|
||||
@@ -202,48 +194,10 @@ impl OtlpMetricCollector {
|
||||
decode_values,
|
||||
fallback_values,
|
||||
decode_error_values,
|
||||
shard_read_values,
|
||||
task,
|
||||
})
|
||||
}
|
||||
|
||||
/// Total physical bytes read from disk across every shard-read label set.
|
||||
async fn shard_read_bytes_total(&self) -> u64 {
|
||||
self.shard_read_values
|
||||
.lock()
|
||||
.await
|
||||
.values()
|
||||
.map(|versions| versions.values().map(|(_, value)| *value).sum::<u64>())
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Waits until the shard-read counter stops advancing so a measurement window
|
||||
/// is not polluted by exports still in flight.
|
||||
///
|
||||
/// Requires several consecutive equal samples spanning more than one export
|
||||
/// interval (`RUSTFS_OBS_METER_INTERVAL=1`): a single unchanged sample only
|
||||
/// proves the latest export has not landed yet, which silently reads as "no
|
||||
/// disk reads happened" and makes any upper-bound assertion vacuous.
|
||||
async fn wait_for_shard_read_bytes_to_settle(&self) -> TestResult<u64> {
|
||||
const REQUIRED_STABLE_SAMPLES: usize = 5;
|
||||
let mut last = self.shard_read_bytes_total().await;
|
||||
let mut stable = 0;
|
||||
for _ in 0..60 {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let current = self.shard_read_bytes_total().await;
|
||||
if current == last {
|
||||
stable += 1;
|
||||
if stable >= REQUIRED_STABLE_SAMPLES {
|
||||
return Ok(current);
|
||||
}
|
||||
} else {
|
||||
stable = 0;
|
||||
last = current;
|
||||
}
|
||||
}
|
||||
Err("timed out waiting for shard-read byte counter to settle".into())
|
||||
}
|
||||
|
||||
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
||||
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
||||
}
|
||||
@@ -367,7 +321,6 @@ async fn handle_metric_export(
|
||||
decode_values: MetricValues,
|
||||
fallback_values: MetricValues,
|
||||
decode_error_values: MetricValues,
|
||||
shard_read_values: MetricValues,
|
||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
if request.uri().path() != "/v1/metrics" {
|
||||
return Ok(response(StatusCode::NOT_FOUND));
|
||||
@@ -401,9 +354,7 @@ async fn handle_metric_export(
|
||||
let mut decode_values = decode_values.lock().await;
|
||||
let mut fallback_values = fallback_values.lock().await;
|
||||
let mut decode_error_values = decode_error_values.lock().await;
|
||||
let mut shard_read_values = shard_read_values.lock().await;
|
||||
record_reader_path_metrics(&export, &mut values);
|
||||
record_shard_read_bytes_metrics(&export, &mut shard_read_values);
|
||||
record_msgpack_decode_metrics(&export, &mut decode_values);
|
||||
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
||||
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
||||
@@ -424,50 +375,6 @@ fn reader_path_metric_key(path: &str, object_class: &str, size_bucket: &str) ->
|
||||
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
||||
}
|
||||
|
||||
/// Accumulates `SHARD_READ_BYTES_COUNTER` across all label sets. Only the total
|
||||
/// matters: it is the number of physical bytes the erasure layer actually pulled
|
||||
/// from disk, which is what separates a bounded per-part read from a decode of
|
||||
/// the whole object.
|
||||
fn record_shard_read_bytes_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||
for resource_metrics in &export.resource_metrics {
|
||||
for scope_metrics in &resource_metrics.scope_metrics {
|
||||
for metric in &scope_metrics.metrics {
|
||||
if metric.name != SHARD_READ_BYTES_COUNTER {
|
||||
continue;
|
||||
}
|
||||
let Some(metric::Data::Sum(sum)) = &metric.data else {
|
||||
continue;
|
||||
};
|
||||
for point in &sum.data_points {
|
||||
let Some(number_data_point::Value::AsInt(value)) = point.value.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let value = u64::try_from(*value).unwrap_or_default();
|
||||
// Keyed by labels, not by position: point order within an export
|
||||
// is not guaranteed stable, so an index key would alias distinct
|
||||
// series across batches.
|
||||
let key = format!(
|
||||
"{}\u{1f}{}\u{1f}{}",
|
||||
attribute_string(&point.attributes, "path").unwrap_or_default(),
|
||||
attribute_string(&point.attributes, "role").unwrap_or_default(),
|
||||
attribute_string(&point.attributes, "outcome").unwrap_or_default(),
|
||||
);
|
||||
values
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.entry(point.start_time_unix_nano)
|
||||
.and_modify(|current| {
|
||||
if point.time_unix_nano >= current.0 {
|
||||
*current = (point.time_unix_nano, value);
|
||||
}
|
||||
})
|
||||
.or_insert((point.time_unix_nano, value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||
for resource_metrics in &export.resource_metrics {
|
||||
for scope_metrics in &resource_metrics.scope_metrics {
|
||||
@@ -1921,122 +1828,39 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
||||
/// Reverting the multipart compression fix must fail this test.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-roundtrip";
|
||||
let bucket = "inline-multipart-compression-fallback";
|
||||
cluster.create_test_bucket(bucket).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
let key = "multipart/compressed.txt";
|
||||
let key = "multipart/compression-disabled.txt";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX),
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A tail range over a compressed multipart object must read only the physical
|
||||
/// data it needs, not decode the object from byte zero.
|
||||
///
|
||||
/// The byte-exactness tests around this one stay green even if the seek path
|
||||
/// regresses into decoding from the start of the object: the bytes returned are
|
||||
/// still correct, only the read amplification explodes. This asserts the cost
|
||||
/// side, using `SHARD_READ_BYTES_COUNTER` — already emitted per shard read by the
|
||||
/// erasure layer, so no production code is instrumented for the test.
|
||||
///
|
||||
/// `get_compressed_offsets` skips whole preceding parts by their stored size and
|
||||
/// then seeks inside the covering part via its compression index, so a bounded
|
||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
||||
/// object.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-tail-range";
|
||||
cluster.create_test_bucket(bucket).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
let key = "multipart/tail-range.txt";
|
||||
let (body, _second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||
|
||||
// Establish that the object really took the compressed read path; otherwise a
|
||||
// small delta below would only prove compression never happened.
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let baseline = collector.wait_for_shard_read_bytes_to_settle().await?;
|
||||
|
||||
let tail_len = 4 * KIB;
|
||||
let start = body.len() - tail_len;
|
||||
let end = body.len() - 1;
|
||||
let range = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range(format!("bytes={start}-{end}"))
|
||||
.send()
|
||||
.await?;
|
||||
let tail = range.body.collect().await?.into_bytes();
|
||||
assert_eq!(tail.as_ref(), &body[start..], "tail range returned wrong bytes");
|
||||
|
||||
let after = collector.wait_for_shard_read_bytes_to_settle().await?;
|
||||
let read_bytes = after.saturating_sub(baseline);
|
||||
|
||||
// A zero delta means the window caught nothing — an unexported counter, or a
|
||||
// read served without touching the erasure layer — which would make the upper
|
||||
// bound vacuously true. Fail instead of passing blind.
|
||||
assert!(
|
||||
read_bytes > 0,
|
||||
"no shard reads observed for the tail range; the budget assertion below would be vacuous"
|
||||
);
|
||||
|
||||
// Part 1 alone is MPU_PART_1_SIZE, so a whole-object decode cannot come in
|
||||
// under it. Half the logical size leaves generous headroom for erasure padding
|
||||
// and unrelated background reads while still failing loudly on a full decode.
|
||||
let budget = (body.len() / 2) as u64;
|
||||
assert!(
|
||||
read_bytes < budget,
|
||||
"tail range read {read_bytes} physical bytes for a {tail_len}-byte range (budget {budget}, object {} bytes): \
|
||||
the read is not bounded to the covering part",
|
||||
body.len()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||
@@ -2047,7 +1871,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
||||
cluster.start().await?;
|
||||
|
||||
@@ -2067,21 +1890,14 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
ReaderPathExpectation::for_class(
|
||||
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
|
||||
LEGACY_DUPLEX,
|
||||
COMPRESSED,
|
||||
MULTIPART,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
assert_part_number_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
PartNumberReaderPathExpectation::new(
|
||||
bucket,
|
||||
multipart_key,
|
||||
&second_part,
|
||||
multipart_body.len(),
|
||||
COMPRESSED,
|
||||
LEGACY_DUPLEX,
|
||||
),
|
||||
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
)
|
||||
.await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||
@@ -2537,11 +2353,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
hot_client.create_bucket().bucket(bucket).send().await?;
|
||||
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
|
||||
|
||||
// `.zip` sits on the disk-compression exclusion list: this test pins
|
||||
// msgpack compat controls across ILM transition, and a compressed object
|
||||
// would classify as `compressed` instead of `remote` (and the warm-tier
|
||||
// read path does not decode compression — tracked separately).
|
||||
let key = "transition/mixed-multipart.zip";
|
||||
let key = "transition/mixed-multipart.bin";
|
||||
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
|
||||
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
|
||||
assert!(
|
||||
|
||||
@@ -1,611 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ILM on SSE-KMS buckets while per-key SSE authorization is enforced (backlog#1582).
|
||||
//!
|
||||
//! Per-key KMS authorization (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true`) scopes the
|
||||
//! SSE-KMS data path to the requesting principal's `kms:GenerateDataKey` /
|
||||
//! `kms:Decrypt` grants. Internal callers — the lifecycle scanner's expiry deletes
|
||||
//! and the tier transition worker's reads — carry no request principal, and
|
||||
//! `authorize_sse_kms_key` (rustfs/src/storage/sse.rs) exempts a `None` principal
|
||||
//! so background maintenance keeps working on encrypted buckets.
|
||||
//!
|
||||
//! These tests pin that exemption end to end. If enforcement ever starts applying
|
||||
//! to the scanner's internal operations, expiry stops happening on SSE-KMS buckets
|
||||
//! and [`ilm_expiration_on_sse_kms_bucket_under_enforcement`] times out; if it
|
||||
//! starts applying to the transition worker or the read-through path,
|
||||
//! [`ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back`] fails at the
|
||||
//! transition wait or the plaintext round-trip.
|
||||
//!
|
||||
//! The replication half of the same acceptance item lives in
|
||||
//! `crates/e2e_test/src/replication_extension_test.rs`
|
||||
//! (`test_bucket_replication_sse_kms_failure_contract`); ILM had no coverage
|
||||
//! before this file.
|
||||
//!
|
||||
//! Deployment constraint pinned by the transition test's setup: the RustFS warm
|
||||
//! backend forwards the object's stored `x-amz-server-side-encryption*` metadata
|
||||
//! as raw headers on the tier data PUT (`build_transition_put_options` +
|
||||
//! `api_put_object.rs` header mapping), so a RustFS tier target must itself have
|
||||
//! KMS enabled and hold the named key or it rejects every transition upload with
|
||||
//! 400 InvalidRequest. That rejection is independent of the enforcement switch;
|
||||
//! the cold server here therefore runs its own Local KMS with the same key id.
|
||||
|
||||
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, RestoreRequest,
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Transition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serial_test::serial;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const SSE_KEY: &str = "kms-ilm-sse-key";
|
||||
const PAYLOAD: &[u8] = b"kms ilm sse payload: survives enforcement, expires and transitions on schedule";
|
||||
|
||||
const EXPIRY_BUCKET: &str = "kms-ilm-expiry";
|
||||
const EXPIRE_KEY: &str = "expire/object.bin";
|
||||
const SURVIVOR_KEY: &str = "keep/object.bin";
|
||||
|
||||
const TIER_NAME: &str = "KMSCOLD";
|
||||
const TIER_BUCKET: &str = "kms-ilm-cold-tier";
|
||||
const TIER_PREFIX: &str = "tiered";
|
||||
const TRANSITION_BUCKET: &str = "kms-ilm-transition";
|
||||
const TRANSITION_KEY: &str = "tier/object.bin";
|
||||
|
||||
/// Generous CI safety net; with a 1s scanner cycle and 2s lifecycle days the
|
||||
/// terminal state normally lands within a few seconds.
|
||||
const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
|
||||
|
||||
/// Start a Local-KMS server with per-key SSE authorization enforced and the
|
||||
/// lifecycle clock accelerated.
|
||||
///
|
||||
/// KMS wiring matches `kms_authorization_negative_matrix_test.rs` (local backend,
|
||||
/// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
|
||||
/// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
|
||||
/// so a `Days=1` rule is due about two seconds after the write.
|
||||
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
|
||||
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
|
||||
|
||||
let key_dir = env.kms_keys_dir.clone();
|
||||
let args = vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
key_dir.as_str(),
|
||||
"--kms-default-key-id",
|
||||
SSE_KEY,
|
||||
];
|
||||
|
||||
let envs = [
|
||||
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
|
||||
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_ILM_PROCESS_TIME", "1"),
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
|
||||
];
|
||||
|
||||
env.base_env.start_rustfs_server_with_env(args, &envs).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the bucket's default encryption to SSE-KMS under [`SSE_KEY`], so plain
|
||||
/// PUTs (and internal rewrites) are encrypted without per-request SSE headers.
|
||||
async fn set_bucket_default_sse_kms(client: &Client, bucket: &str) -> TestResult {
|
||||
let encryption_config = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::AwsKms)
|
||||
.kms_master_key_id(SSE_KEY)
|
||||
.build()?,
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(encryption_config)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Assert via `HeadObject` that the stored object is SSE-KMS encrypted under
|
||||
/// [`SSE_KEY`]. Without this, a bucket-default misconfiguration would let the
|
||||
/// tests pass on an unencrypted object and prove nothing about KMS.
|
||||
async fn assert_head_sse_kms(client: &Client, bucket: &str, key: &str) -> TestResult {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(
|
||||
head.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"{bucket}/{key} must be SSE-KMS encrypted via the bucket default"
|
||||
);
|
||||
assert_eq!(
|
||||
head.ssekms_key_id(),
|
||||
Some(SSE_KEY),
|
||||
"{bucket}/{key} must be wrapped under the configured KMS key"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns `true` once `GET bucket/key` fails with `NoSuchKey`, `false` while it
|
||||
/// still succeeds. Any other error is surfaced. (Copied from
|
||||
/// `reliant/lifecycle.rs`; that helper is private to the reliant module.)
|
||||
async fn object_is_gone(client: &Client, bucket: &str, key: &str) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
|
||||
match client.get_object().bucket(bucket).key(key).send().await {
|
||||
Ok(output) => {
|
||||
output.body.collect().await?;
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(service_error) = e.as_service_error() {
|
||||
if service_error.is_no_such_key() {
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(format!("expected NoSuchKey, got: {e:?}").into());
|
||||
}
|
||||
Err(format!("expected a service error, got: {e:?}").into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll until `GET bucket/key` returns `NoSuchKey`, or fail after `deadline`.
|
||||
async fn wait_for_object_expired(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
if object_is_gone(client, bucket, key).await? {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} was not expired by the lifecycle scanner within {}s; \
|
||||
SSE key-policy enforcement may have started blocking the scanner's internal deletes",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a prefix-scoped `Days`-based expiration rule.
|
||||
async fn put_expiration_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.expiration(LifecycleExpiration::builder().days(days).build())
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Install a prefix-scoped `Days`-based transition rule targeting [`TIER_NAME`].
|
||||
async fn put_transition_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.transitions(
|
||||
Transition::builder()
|
||||
.days(days)
|
||||
.storage_class(TransitionStorageClass::from(TIER_NAME))
|
||||
.build(),
|
||||
)
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a plain Local-KMS server (no enforcement, no lifecycle acceleration)
|
||||
/// holding [`SSE_KEY`], to serve as the cold tier target.
|
||||
///
|
||||
/// The RustFS warm backend forwards the stored SSE-KMS headers on the tier data
|
||||
/// PUT, so the target re-applies managed SSE-KMS under the named key and must
|
||||
/// be able to resolve it; without KMS it answers 400 InvalidRequest and the
|
||||
/// transition can never complete. Enforcement stays off here: the tier writes
|
||||
/// arrive under `cold`'s root credentials, and one enforcing side is enough to
|
||||
/// pin the exemption.
|
||||
async fn start_cold_tier_kms_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
|
||||
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
|
||||
|
||||
let key_dir = env.kms_keys_dir.clone();
|
||||
let args = vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
key_dir.as_str(),
|
||||
"--kms-default-key-id",
|
||||
SSE_KEY,
|
||||
];
|
||||
|
||||
env.base_env
|
||||
.start_rustfs_server_with_env(args, &[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The subset of the manual transition run report these tests assert on.
|
||||
///
|
||||
/// Unknown fields are ignored, so this stays compatible with report growth; the
|
||||
/// full shape is pinned by `reliant/tiering.rs`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManualTransitionRunReport {
|
||||
#[serde(default)]
|
||||
scanned: u64,
|
||||
#[serde(default)]
|
||||
enqueued: u64,
|
||||
#[serde(default)]
|
||||
skipped_already_in_flight: u64,
|
||||
#[serde(default)]
|
||||
skipped_tier: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManualTransitionRunResponse {
|
||||
state: String,
|
||||
report: ManualTransitionRunReport,
|
||||
}
|
||||
|
||||
/// One synchronous (enqueue-only) manual transition run over `bucket/prefix`,
|
||||
/// via the same admin endpoint `reliant/tiering.rs` drives.
|
||||
async fn manual_transition_run(
|
||||
hot: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> Result<ManualTransitionRunResponse, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let bucket = urlencoding::encode(bucket);
|
||||
let prefix = urlencoding::encode(prefix);
|
||||
let tier = urlencoding::encode(TIER_NAME);
|
||||
let path =
|
||||
format!("/rustfs/admin/v3/ilm/transition/run?bucket={bucket}&prefix={prefix}&tier={tier}&dryRun=false&maxObjects=10");
|
||||
let (status, body) = admin_request(&hot.url, http::Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("manual transition run failed: status={status}, body={body}").into());
|
||||
}
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
/// Drive manual transition runs until one reports the object as processed.
|
||||
///
|
||||
/// The `Days=1` rule becomes due about two seconds after the write
|
||||
/// (`RUSTFS_ILM_DEBUG_DAY_SECS=2`), so early runs may legitimately report the
|
||||
/// object as not yet eligible; the loop keeps running the endpoint until it
|
||||
/// either enqueues the transition, sees it already in flight (the 1s scanner
|
||||
/// backstop got there first), or finds it already on the tier.
|
||||
async fn run_manual_transition_until_processed(
|
||||
hot: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
deadline: StdDuration,
|
||||
) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let run = manual_transition_run(hot, bucket, prefix).await?;
|
||||
assert_eq!(run.report.scanned, 1, "manual transition run must scan the object: {run:#?}");
|
||||
if run.report.enqueued + run.report.skipped_already_in_flight + run.report.skipped_tier >= 1 {
|
||||
info!(state = %run.state, report = ?run.report, "manual transition run processed the SSE-KMS object");
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"manual transition runs never processed {bucket}/{prefix} within {}s; last report: {run:#?}",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
|
||||
///
|
||||
/// No `force`, so the server runs the real connectivity probe against `cold`
|
||||
/// (the tier bucket must already exist there). Mirrors
|
||||
/// `reliant/tiering.rs::add_rustfs_tier`, which is private to that module.
|
||||
async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
|
||||
let body = serde_json::json!({
|
||||
"type": "rustfs",
|
||||
"rustfs": {
|
||||
"name": TIER_NAME,
|
||||
"endpoint": cold.url.as_str(),
|
||||
"accessKey": cold.access_key.as_str(),
|
||||
"secretKey": cold.secret_key.as_str(),
|
||||
"bucket": TIER_BUCKET,
|
||||
"prefix": TIER_PREFIX,
|
||||
"region": "us-east-1",
|
||||
"storageClass": ""
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let (status, resp) = admin_request(
|
||||
&hot.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/tier",
|
||||
Some(body),
|
||||
&hot.access_key,
|
||||
&hot.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Poll `HEAD` until the object's storage class is the tier name (transition
|
||||
/// complete), or fail after `deadline`. (From `reliant/tiering.rs`.)
|
||||
async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
if head.storage_class().map(|sc| sc.as_str()) == Some(TIER_NAME) {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} was not transitioned to {TIER_NAME} within {}s (storage_class={:?}); \
|
||||
SSE key-policy enforcement may have started blocking the transition worker's internal reads",
|
||||
deadline.as_secs(),
|
||||
head.storage_class()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll `HEAD` until `x-amz-restore` reports a finished restore
|
||||
/// (`ongoing-request="false"`), or fail after `deadline`.
|
||||
async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
if head.restore().is_some_and(|r| r.contains("ongoing-request=\"false\"")) {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} restore did not complete within {}s (restore={:?}); \
|
||||
SSE key-policy enforcement may have started blocking the restore copy-back's internal reads",
|
||||
deadline.as_secs(),
|
||||
head.restore()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// ILM expiration keeps working on an SSE-KMS bucket while per-key SSE
|
||||
/// authorization is enforced.
|
||||
///
|
||||
/// The lifecycle scanner deletes expired objects with an internal (no-principal)
|
||||
/// identity that holds no `kms` grant. If enforcement ever starts applying to
|
||||
/// those internal deletes (or to the scanner's metadata reads) on encrypted
|
||||
/// buckets, expiry stops happening and this test times out.
|
||||
///
|
||||
/// A survivor object under a non-matching prefix isolates the rule's prefix
|
||||
/// filter as the cause of the deletion and proves the encrypted bucket stays
|
||||
/// readable end to end after the scanner has run.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
|
||||
|
||||
let client = env.base_env.create_s3_client();
|
||||
set_bucket_default_sse_kms(&client, EXPIRY_BUCKET).await?;
|
||||
|
||||
for key in [EXPIRE_KEY, SURVIVOR_KEY] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(EXPIRY_BUCKET)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(PAYLOAD))
|
||||
.send()
|
||||
.await?;
|
||||
assert_head_sse_kms(&client, EXPIRY_BUCKET, key).await?;
|
||||
}
|
||||
info!("both objects stored SSE-KMS encrypted under enforcement");
|
||||
|
||||
put_expiration_rule(&client, EXPIRY_BUCKET, "kms-ilm-expire", "expire/", 1).await?;
|
||||
|
||||
// The regression this pins: the scanner's internal delete must stay exempt
|
||||
// from per-key SSE authorization, so the encrypted object actually expires.
|
||||
wait_for_object_expired(&client, EXPIRY_BUCKET, EXPIRE_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object expired by the lifecycle scanner under enforcement");
|
||||
|
||||
// Negative control: same bucket, same encryption, non-matching prefix. It
|
||||
// must survive the scanner and still decrypt for the requesting principal.
|
||||
assert!(
|
||||
!object_is_gone(&client, EXPIRY_BUCKET, SURVIVOR_KEY).await?,
|
||||
"non-matching-prefix object must not be expired by a prefix-scoped rule"
|
||||
);
|
||||
let survivor = client.get_object().bucket(EXPIRY_BUCKET).key(SURVIVOR_KEY).send().await?;
|
||||
assert_eq!(
|
||||
survivor.body.collect().await?.into_bytes().as_ref(),
|
||||
PAYLOAD,
|
||||
"surviving SSE-KMS object must still decrypt after the scanner has run"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// ILM transition to a remote tier keeps working on an SSE-KMS bucket while
|
||||
/// per-key SSE authorization is enforced, and the transitioned object reads
|
||||
/// back as plaintext.
|
||||
///
|
||||
/// The transition worker moves the stored (encrypted) bytes to the cold tier
|
||||
/// with an internal (no-principal) identity; the read-through `GET` then
|
||||
/// decrypts the envelope for the requesting principal. If enforcement ever
|
||||
/// starts applying to the worker's internal reads, the transition wait times
|
||||
/// out; if the stored envelope is mishandled across the tier round trip, the
|
||||
/// plaintext comparison fails.
|
||||
///
|
||||
/// The transition is driven through the manual transition-run admin endpoint
|
||||
/// (the mechanism `reliant/tiering.rs` established), so the test does not
|
||||
/// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
// Cold-tier server: independent credentials, its own Local KMS holding the
|
||||
// same key id (see the module docs for why the tier target needs KMS).
|
||||
// Started first; each server's startup cleanup only matches its own unique
|
||||
// address and temp dir, so the two instances coexist.
|
||||
let mut cold = LocalKMSTestEnvironment::new().await?;
|
||||
cold.base_env.access_key = "kmscoldtieradmin".to_string();
|
||||
cold.base_env.secret_key = "kmscoldtiersecret".to_string();
|
||||
start_cold_tier_kms_server(&mut cold).await?;
|
||||
let cold_client = cold.base_env.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
// Hot server: Local KMS + enforcement + accelerated lifecycle clock.
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
let hot_client = env.base_env.create_s3_client();
|
||||
|
||||
add_rustfs_tier(&env.base_env, &cold.base_env).await?;
|
||||
|
||||
env.base_env.create_test_bucket(TRANSITION_BUCKET).await?;
|
||||
set_bucket_default_sse_kms(&hot_client, TRANSITION_BUCKET).await?;
|
||||
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.body(ByteStream::from_static(PAYLOAD))
|
||||
.send()
|
||||
.await?;
|
||||
assert_head_sse_kms(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY).await?;
|
||||
info!("object stored SSE-KMS encrypted under enforcement");
|
||||
|
||||
// Days=1 is due ~2s after the write with RUSTFS_ILM_DEBUG_DAY_SECS=2.
|
||||
put_transition_rule(&hot_client, TRANSITION_BUCKET, "kms-ilm-transition", "tier/", 1).await?;
|
||||
|
||||
// Drive the transition deterministically via the manual run endpoint, then
|
||||
// wait for HEAD to report the tier as the object's storage class.
|
||||
run_manual_transition_until_processed(&env.base_env, TRANSITION_BUCKET, "tier/", ILM_DEADLINE).await?;
|
||||
wait_for_transition(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object transitioned to the remote tier under enforcement");
|
||||
|
||||
let head = hot_client
|
||||
.head_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
head.restore().is_none(),
|
||||
"a freshly transitioned object must not advertise x-amz-restore, got {:?}",
|
||||
head.restore()
|
||||
);
|
||||
|
||||
// The remote copy exists on the cold tier. The payload the tier holds is the
|
||||
// hot server's stored ciphertext, wrapped once more under the cold server's
|
||||
// own managed SSE-KMS layer (the forwarded headers re-request encryption).
|
||||
let remote = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
|
||||
assert!(!remote.contents().is_empty(), "cold-tier bucket must hold the transitioned object's data");
|
||||
|
||||
// Read-through GET under enforcement must succeed (not AccessDenied) and
|
||||
// keep advertising SSE-KMS. Its BODY is deliberately not compared here:
|
||||
// the transitioned read path skips managed-SSE decryption — a product gap
|
||||
// unrelated to enforcement — so a direct GET streams the stored ciphertext
|
||||
// (`new_getobjectreader` in crates/ecstore/src/client/object_api_utils.rs
|
||||
// hardcodes `is_encrypted = false` and never applies the
|
||||
// `ReadTransform::Encrypted` wrapping the hot-read path builds in
|
||||
// crates/ecstore/src/object_api/readers.rs). Plaintext recovery is pinned
|
||||
// through restore semantics below; when the read-through gap is fixed, a
|
||||
// byte assertion can be added here too.
|
||||
let read_through = hot_client
|
||||
.get_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
read_through.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"transitioned object must still report SSE-KMS on read-through"
|
||||
);
|
||||
let read_through_body = read_through.body.collect().await?.into_bytes();
|
||||
assert_eq!(
|
||||
read_through_body.len(),
|
||||
PAYLOAD.len(),
|
||||
"read-through GET must stream the object's full logical size under enforcement"
|
||||
);
|
||||
|
||||
// RestoreObject copies the ciphertext back from the tier under the original
|
||||
// envelope metadata; the restored copy is then served by the normal
|
||||
// decrypting read path. The copy-back runs with an internal (no-principal)
|
||||
// identity, so this also pins the exemption on the restore path. Days=300
|
||||
// because RUSTFS_ILM_DEBUG_DAY_SECS=2 accelerates the restored copy's
|
||||
// expiry as well (300 accelerated days == 600s of validity).
|
||||
hot_client
|
||||
.restore_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.restore_request(RestoreRequest::builder().days(300).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_restore_complete(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
|
||||
info!("SSE-KMS object restored from the remote tier under enforcement");
|
||||
|
||||
// The KMS-relevant half: the restored envelope decrypts back to the exact
|
||||
// plaintext for the requesting principal.
|
||||
let restored = hot_client
|
||||
.get_object()
|
||||
.bucket(TRANSITION_BUCKET)
|
||||
.key(TRANSITION_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
restored.server_side_encryption(),
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"restored object must still report SSE-KMS"
|
||||
);
|
||||
let body = restored.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), PAYLOAD, "restored SSE-KMS object must round-trip byte-identical plaintext");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -59,6 +59,3 @@ mod configured_roundtrip_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod kms_authorization_negative_matrix_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod kms_ilm_sse_kms_test;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,6 +33,7 @@ use aws_sdk_s3::types::{
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -106,6 +107,7 @@ fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc>
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by COMPLIANCE retention");
|
||||
@@ -143,6 +145,7 @@ async fn test_delete_object_blocked_by_compliance_retention() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_governance_without_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by GOVERNANCE retention without bypass");
|
||||
@@ -172,6 +175,7 @@ async fn test_delete_object_blocked_by_governance_without_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_allowed_by_governance_with_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject allowed by GOVERNANCE retention with bypass");
|
||||
@@ -211,6 +215,7 @@ async fn test_delete_object_allowed_by_governance_with_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_creates_delete_marker_for_retained_current_version() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject creates delete marker for retained current version");
|
||||
@@ -261,6 +266,7 @@ async fn test_delete_object_creates_delete_marker_for_retained_current_version()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by Legal Hold");
|
||||
@@ -293,6 +299,7 @@ async fn test_delete_object_blocked_by_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_allowed_with_legal_hold_off() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject allowed with Legal Hold OFF");
|
||||
@@ -328,6 +335,7 @@ async fn test_delete_object_allowed_with_legal_hold_off() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_after_legal_hold_removed() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject succeeds after Legal Hold is removed");
|
||||
@@ -361,6 +369,7 @@ async fn test_delete_object_after_legal_hold_removed() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_legal_hold_returns_updated_status() {
|
||||
init_logging();
|
||||
info!("🧪 Test: GetObjectLegalHold returns updated status");
|
||||
@@ -416,6 +425,7 @@ async fn test_get_object_legal_hold_returns_updated_status() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_retention_returns_configured_values() {
|
||||
init_logging();
|
||||
info!("🧪 Test: GetObjectRetention returns configured values");
|
||||
@@ -466,6 +476,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
// creating a new current version. The lock protects the existing version
|
||||
// from deletion; it never blocks new versions.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObject overwrite of a legal-hold version creates a new version");
|
||||
@@ -550,6 +561,7 @@ async fn test_put_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_applies_requested_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject applies requested Legal Hold");
|
||||
@@ -601,6 +613,7 @@ async fn test_copy_object_applies_requested_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_does_not_inherit_source_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject does not inherit source Legal Hold");
|
||||
@@ -694,6 +707,7 @@ async fn test_copy_object_does_not_inherit_source_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject overwrite of a legal-hold destination creates a new version");
|
||||
@@ -773,6 +787,7 @@ async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_create_multipart_upload_applies_requested_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CreateMultipartUpload applies requested Legal Hold");
|
||||
@@ -838,6 +853,7 @@ async fn test_create_multipart_upload_applies_requested_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_create_multipart_upload_creates_new_version_under_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CreateMultipartUpload over a COMPLIANCE-retained key creates a new version");
|
||||
@@ -917,6 +933,7 @@ async fn test_create_multipart_upload_creates_new_version_under_compliance_reten
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Delete completed multipart object blocked by Legal Hold");
|
||||
@@ -976,6 +993,7 @@ async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_completed_multipart_object_blocked_by_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Delete completed multipart object blocked by retention");
|
||||
@@ -1037,6 +1055,7 @@ async fn test_delete_completed_multipart_object_blocked_by_retention() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under Legal Hold");
|
||||
@@ -1116,6 +1135,7 @@ async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_complete_multipart_upload_creates_new_version_under_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under COMPLIANCE retention");
|
||||
@@ -1189,6 +1209,7 @@ async fn test_complete_multipart_upload_creates_new_version_under_compliance_ret
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_paths_require_put_object_legal_hold_permission() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths require PutObjectLegalHold permission");
|
||||
@@ -1252,6 +1273,7 @@ async fn test_write_paths_require_put_object_legal_hold_permission() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_paths_require_put_object_retention_permission() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths require PutObjectRetention permission");
|
||||
@@ -1323,6 +1345,7 @@ async fn test_write_paths_require_put_object_retention_permission() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_objects_mixed_locked_unlocked() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObjects with mixed locked and unlocked objects");
|
||||
@@ -1404,6 +1427,7 @@ async fn test_delete_objects_mixed_locked_unlocked() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_compliance_cannot_shorten() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention cannot shorten COMPLIANCE retention");
|
||||
@@ -1444,6 +1468,7 @@ async fn test_put_retention_compliance_cannot_shorten() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_compliance_can_extend() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention can extend COMPLIANCE retention");
|
||||
@@ -1484,6 +1509,7 @@ async fn test_put_retention_compliance_can_extend() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_governance_extend_without_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention on GOVERNANCE can extend without bypass");
|
||||
@@ -1527,6 +1553,7 @@ async fn test_put_retention_governance_extend_without_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_governance_shorten_requires_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention on GOVERNANCE requires bypass to shorten");
|
||||
@@ -1588,6 +1615,7 @@ async fn test_put_retention_governance_shorten_requires_bypass() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_default_retention_applied_to_new_objects() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Default retention is applied to new objects");
|
||||
@@ -1657,6 +1685,7 @@ async fn test_default_retention_applied_to_new_objects() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_creates_delete_marker_for_default_retained_current_version() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject creates delete marker for default-retained current version");
|
||||
@@ -1741,6 +1770,7 @@ async fn test_delete_object_creates_delete_marker_for_default_retained_current_v
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths reject incomplete Object Lock retention headers");
|
||||
@@ -1839,6 +1869,7 @@ async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_retention_uses_destination_policy() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject retention follows destination policy");
|
||||
@@ -2020,6 +2051,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_default_retention_fixed_at_create() {
|
||||
init_logging();
|
||||
info!("🧪 Test: multipart default retention is fixed at CreateMultipartUpload");
|
||||
@@ -2090,6 +2122,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)");
|
||||
@@ -2210,6 +2243,7 @@ async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioning_auto_enabled_with_object_lock() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Versioning is auto-enabled when Object Lock is configured");
|
||||
@@ -2268,6 +2302,7 @@ async fn test_versioning_auto_enabled_with_object_lock() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_error_message_distinguishes_legal_hold_from_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Error messages distinguish Legal Hold from Retention");
|
||||
|
||||
@@ -2854,7 +2854,7 @@ pub(crate) mod cmptst_30 {
|
||||
result
|
||||
}
|
||||
|
||||
#[ignore = "timing-sensitive backend-pressure latency probe; run explicitly with --ignored"]
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn regression() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
crate::common::init_logging();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,11 +32,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
# RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
|
||||
# state in a stock build (backlog#1832).
|
||||
list-chaos = []
|
||||
rio-v2 = ["dep:rustfs-rio-v2"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
@@ -273,7 +268,6 @@ proptest = "1"
|
||||
rcgen.workspace = true
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
tonic-prost = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] }
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,11 +61,9 @@ pub mod bucket {
|
||||
delete_manual_transition_scope_admission_if_current, load_manual_transition_job_record,
|
||||
load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
|
||||
manual_transition_job_lease_expired, manual_transition_scope_admission_lease_expired,
|
||||
manual_transition_scope_key, persist_manual_transition_job_progress,
|
||||
persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease,
|
||||
renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel,
|
||||
save_manual_transition_job_record, save_manual_transition_job_record_if_current,
|
||||
save_manual_transition_scope_admission_if_absent, update_manual_transition_job_record,
|
||||
manual_transition_scope_key, persist_manual_transition_job_progress, renew_manual_transition_job_lease,
|
||||
request_manual_transition_job_cancel, save_manual_transition_job_record,
|
||||
save_manual_transition_job_record_if_current, save_manual_transition_scope_admission_if_absent,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -131,19 +129,16 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -185,26 +180,24 @@ pub mod bucket {
|
||||
mrf_backlog_observability_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -281,9 +274,7 @@ pub mod cluster {
|
||||
}
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled};
|
||||
}
|
||||
|
||||
pub mod config {
|
||||
@@ -323,7 +314,7 @@ pub mod data_usage {
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||
@@ -353,7 +344,7 @@ pub mod disk {
|
||||
}
|
||||
|
||||
pub mod error {
|
||||
pub use crate::disk::error::{DiskError, Error, FileAccessDeniedWithContext, Result};
|
||||
pub use crate::disk::error::{BitrotErrorType, DiskError, Error, FileAccessDeniedWithContext, Result};
|
||||
}
|
||||
|
||||
pub mod error_reduce {
|
||||
@@ -374,14 +365,14 @@ pub mod error {
|
||||
|
||||
pub mod erasure {
|
||||
pub use crate::erasure::coding::{
|
||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||
calc_shard_size, calc_shard_size_legacy,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod event {
|
||||
pub use crate::event::name::EventName;
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||
}
|
||||
|
||||
pub mod global {
|
||||
@@ -410,11 +401,8 @@ pub mod metrics {
|
||||
}
|
||||
|
||||
pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
@@ -474,8 +462,7 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -484,7 +471,6 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
|
||||
@@ -27,15 +27,10 @@ use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
use aws_sdk_s3::operation::delete_object_tagging::{DeleteObjectTaggingError, DeleteObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
};
|
||||
@@ -62,10 +57,8 @@ use rustfs_utils::http::{
|
||||
is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||
};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -87,6 +80,7 @@ use tracing::warn;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
@@ -299,41 +293,9 @@ struct TargetClientBuildProbe {
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
/// SSE-C passthrough capability verdicts (see the enum's own docs in
|
||||
/// `rustfs-replication`) are cached here per target ARN: entries follow the
|
||||
/// `arn_remotes_map` lifecycle (rebuilding or removing a target resets its
|
||||
/// capability to `Unknown`) and additionally expire after
|
||||
/// [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], after which the next attempt
|
||||
/// re-audits. Re-exported so existing `bucket_target_sys` consumers keep
|
||||
/// their import path while the verdict vocabulary lives with the
|
||||
/// replication decision logic.
|
||||
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
/// Trade-off: without a TTL a verdict is sticky for the process lifetime —
|
||||
/// an `Unsupported` target that gets upgraded (or re-probed only via
|
||||
/// replication-check) would keep failing SSE-C replication forever, and the
|
||||
/// fail-open twin: a `Supported` verdict would outlive a backend swapped
|
||||
/// behind the same endpoint/ARN. With the TTL, a bad target costs at most
|
||||
/// one wasted PUT+HEAD audit per TTL window, and a changed backend is
|
||||
/// re-discovered within the same window.
|
||||
pub const SSEC_PASSTHROUGH_CAPABILITY_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
/// A recorded SSE-C passthrough verdict plus when it was recorded, so reads
|
||||
/// can report staleness against [`SSEC_PASSTHROUGH_CAPABILITY_TTL`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct SsecPassthroughRecord {
|
||||
capability: SsecPassthroughCapability,
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
@@ -354,7 +316,6 @@ impl BucketTargetSys {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -618,59 +579,19 @@ impl BucketTargetSys {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
|
||||
if let Some(targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
/// Staleness is computed here so the gate policy stays a pure function.
|
||||
pub async fn ssec_passthrough_capability(&self, arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
match self.ssec_passthrough_map.read().await.get(arn) {
|
||||
Some(record) => (record.capability, record.recorded_at.elapsed() >= SSEC_PASSTHROUGH_CAPABILITY_TTL),
|
||||
None => (SsecPassthroughCapability::Unknown, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record an audited SSE-C passthrough verdict for a target ARN. Written by
|
||||
/// the replication worker's HEAD-back audit and by the replication-check
|
||||
/// SsecPassthrough probe phase.
|
||||
pub async fn record_ssec_passthrough_capability(&self, arn: &str, capability: SsecPassthroughCapability) {
|
||||
self.ssec_passthrough_map.write().await.insert(
|
||||
arn.to_string(),
|
||||
SsecPassthroughRecord {
|
||||
capability,
|
||||
recorded_at: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Test hook: age an existing verdict so TTL expiry is observable without
|
||||
/// waiting out the real window.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn backdate_ssec_passthrough_capability(&self, arn: &str, age: Duration) {
|
||||
let backdated = Instant::now()
|
||||
.checked_sub(age)
|
||||
.expect("system uptime must exceed the backdate age");
|
||||
if let Some(record) = self.ssec_passthrough_map.write().await.get_mut(arn) {
|
||||
record.recorded_at = backdated;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set_target(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1026,21 +947,15 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1530,43 +1445,6 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the S3 `versionId` for a proxied read against a remote target.
|
||||
/// RustFS represents the null version internally as the nil UUID while the S3
|
||||
/// API addresses it as the literal "null" (same mapping as
|
||||
/// [`resolve_put_api_version_id`]); empty means "no version requested".
|
||||
pub(crate) fn resolve_read_api_version_id(version_id: Option<String>) -> Option<String> {
|
||||
let version_id = version_id?;
|
||||
let trimmed = version_id.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else if Uuid::parse_str(trimmed).is_ok_and(|uuid| uuid.is_nil()) {
|
||||
Some(rustfs_filemeta::NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Outbound header set for a proxied read: the caller-provided passthrough
|
||||
/// headers (client SSE-C key family, conditional headers) plus the anti-loop
|
||||
/// `source-proxy-request` marker in both the x-rustfs- and x-minio- prefixes
|
||||
/// (a MinIO target only understands the latter). Never adds
|
||||
/// `source-replication-check`: that exemption channel belongs exclusively to
|
||||
/// the replication worker's HEAD.
|
||||
fn proxy_outbound_headers(mut extra_headers: HeaderMap) -> HeaderMap {
|
||||
insert_header(&mut extra_headers, SUFFIX_SOURCE_PROXY_REQUEST, "true");
|
||||
extra_headers
|
||||
}
|
||||
|
||||
/// Copy `headers` onto an SDK request inside `customize().map_request` (runs
|
||||
/// before signing, so the headers join the SigV4 canonical request).
|
||||
fn apply_extra_headers(mut req: HttpRequest, headers: &HeaderMap) -> Result<HttpRequest, std::convert::Infallible> {
|
||||
for (k, v) in headers.iter() {
|
||||
req.headers_mut()
|
||||
.insert(k.as_str().to_string(), v.to_str().unwrap_or("").to_string());
|
||||
}
|
||||
Ok(req)
|
||||
}
|
||||
|
||||
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
||||
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
@@ -1598,12 +1476,9 @@ impl Default for AdvancedPutOptions {
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
source_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||
// timestamp header for it, otherwise a receiver would treat an
|
||||
// unset category as a modification made right now.
|
||||
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
retention_timestamp: OffsetDateTime::now_utc(),
|
||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
@@ -1670,8 +1545,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1682,7 +1557,6 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1801,16 +1675,6 @@ impl PutObjectOptions {
|
||||
);
|
||||
}
|
||||
|
||||
for (suffix, timestamp) in [
|
||||
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||
] {
|
||||
if timestamp.unix_timestamp() != 0 {
|
||||
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
|
||||
if self.internal.replication_request {
|
||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
@@ -1818,7 +1682,6 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
@@ -1974,13 +1837,6 @@ impl TargetClient {
|
||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
// `source-proxy-request: false` (MinIO `ProxyHeaderSet` semantics):
|
||||
// the header's mere presence tells the receiver to answer LOCALLY
|
||||
// instead of proxying the miss back to us. Without it, a not-found on
|
||||
// the target gets read-proxied back to this source, echoes the source
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
match self
|
||||
.client
|
||||
.head_object()
|
||||
@@ -2005,129 +1861,6 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
/// Deliberately different from [`TargetClient::head_object`]: it must NOT
|
||||
/// send `source-replication-check` — that header is the replication
|
||||
/// worker's SSE-C metadata exemption channel. A proxied client request
|
||||
/// instead forwards the client's own SSE-C headers (`extra_headers`) so
|
||||
/// the target performs the real SSE-C validation/decryption. The
|
||||
/// `source-proxy-request` marker is always added so the target does not
|
||||
/// proxy the request onward (anti-loop).
|
||||
pub async fn head_object_for_proxy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||
/// Returns the streaming SDK output; callers must forward the body without
|
||||
/// buffering it. Same header contract as [`Self::head_object_for_proxy`]:
|
||||
/// anti-loop marker on, replication-check never sent, client SSE-C /
|
||||
/// conditional headers forwarded verbatim via `extra_headers`.
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GetObjectTagging for the tagging read-proxy path
|
||||
/// (MinIO `proxyGetTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn get_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// PutObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn put_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
tagging: SdkTagging,
|
||||
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.tagging(tagging)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn delete_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.delete_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
@@ -2757,57 +2490,6 @@ mod tests {
|
||||
assert_eq!(health.last_online, Some(now));
|
||||
}
|
||||
|
||||
/// N2 TTL contract, both flip directions: a recorded verdict is fresh
|
||||
/// until [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], then reads as expired; a
|
||||
/// re-audit that records the OPPOSITE verdict replaces it as fresh. The
|
||||
/// worker gate maps expired verdicts to ProceedWithAudit (pinned in
|
||||
/// `replication_target_boundary`), so together this proves an Unsupported
|
||||
/// target recovers to Supported through the audit once its verdict ages
|
||||
/// out — and a stale Supported one is re-proven rather than trusted.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_capability_ttl_expires_and_reaudit_flips_verdict() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:ssec-ttl";
|
||||
let expired_age = SSEC_PASSTHROUGH_CAPABILITY_TTL + Duration::from_secs(1);
|
||||
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unknown, false),
|
||||
"an unrecorded target must read Unknown and never expired"
|
||||
);
|
||||
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Unsupported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, false)
|
||||
);
|
||||
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, true),
|
||||
"an aged-out Unsupported verdict must read expired so the gate re-audits"
|
||||
);
|
||||
|
||||
// The re-audit against an upgraded target records Supported afresh.
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Supported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, false),
|
||||
"a fresh Supported verdict replaces the expired Unsupported one"
|
||||
);
|
||||
|
||||
// And the fail-open twin: Supported also ages out.
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, true),
|
||||
"an aged-out Supported verdict must read expired so the gate re-proves it"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
||||
let sys = BucketTargetSys::default();
|
||||
@@ -3160,57 +2842,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||
// last-writer-wins on these headers (object-api-options.go parses them
|
||||
// as RFC3339); a replica without them loses every conflict resolution.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||
opts.internal.tagging_timestamp = tagging;
|
||||
opts.internal.retention_timestamp = retention;
|
||||
opts.internal.legalhold_timestamp = legalhold;
|
||||
|
||||
let header = opts.header();
|
||||
for (suffix, expected) in [
|
||||
("source-replication-tagging-timestamp", tagging),
|
||||
("source-replication-retention-timestamp", retention),
|
||||
("source-replication-legalhold-timestamp", legalhold),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||
"replication put requests must carry the {suffix} header"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||
// make the receiver treat an unset category as a fresh modification.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
|
||||
let header = opts.header();
|
||||
for suffix in [
|
||||
"source-replication-tagging-timestamp",
|
||||
"source-replication-retention-timestamp",
|
||||
"source-replication-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||
"unset {suffix} must not be sent to replication targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,21 +86,6 @@ where
|
||||
com::save_config_with_opts(api, file, data, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config_with_opts_quiet<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::save_config_with_opts_quiet(api, file, data, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -45,104 +45,6 @@ const MANUAL_TRANSITION_JOB_LEASE_SECONDS: i128 = 60;
|
||||
const MANUAL_TRANSITION_LEGACY_SCOPE_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_TASK_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_WORKER_RESULT_SCAN_LIMIT: i32 = 1000;
|
||||
const MANUAL_TRANSITION_JOB_CAS_RETRIES: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
struct ManualTransitionJobCasBarrierState {
|
||||
job_id: Uuid,
|
||||
paused: std::sync::atomic::AtomicBool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Semaphore,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ManualTransitionJobCasBarrier {
|
||||
state: Arc<ManualTransitionJobCasBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static MANUAL_TRANSITION_JOB_CAS_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<ManualTransitionJobCasBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl ManualTransitionJobCasBarrier {
|
||||
pub(crate) fn install(job_id: Uuid) -> Self {
|
||||
let state = Arc::new(ManualTransitionJobCasBarrierState {
|
||||
job_id,
|
||||
paused: std::sync::atomic::AtomicBool::new(false),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
});
|
||||
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison");
|
||||
assert!(
|
||||
slot.is_none(),
|
||||
"manual transition job CAS barrier must be installed by one test at a time"
|
||||
);
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
if self.state.paused.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
arrived.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("manual transition job update should reach the deterministic CAS barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.add_permits(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ManualTransitionJobCasBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_manual_transition_job_before_first_cas(job_id: Uuid) {
|
||||
let barrier = MANUAL_TRANSITION_JOB_CAS_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("manual transition progress CAS barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.job_id == job_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier
|
||||
&& barrier
|
||||
.paused
|
||||
.compare_exchange(false, true, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
barrier.arrived.notify_one();
|
||||
barrier
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("manual transition job CAS barrier should remain open")
|
||||
.forget();
|
||||
}
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
@@ -246,6 +148,7 @@ impl ManualTransitionJobRecord {
|
||||
|
||||
pub fn fail(&mut self, error: impl Into<String>) {
|
||||
self.state = ManualTransitionJobState::Failed;
|
||||
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
|
||||
self.error = Some(error.into());
|
||||
self.mark_updated_terminal();
|
||||
}
|
||||
@@ -759,10 +662,6 @@ pub struct ManualTransitionWorkerResultRecord {
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultRecord {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||
Self::new_with_reason(job_id, task_key, result, None)
|
||||
}
|
||||
@@ -1141,7 +1040,7 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
}
|
||||
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
|
||||
let data = job.encode().map_err(manual_transition_job_store_error)?;
|
||||
config_boundary::save_config_with_opts_quiet(
|
||||
config_boundary::save_config_with_opts(
|
||||
api,
|
||||
&object,
|
||||
data,
|
||||
@@ -1157,54 +1056,6 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Applies a job-record mutation with optimistic concurrency control.
|
||||
///
|
||||
/// The mutation returns whether the record needs to be persisted. When a lease
|
||||
/// is supplied, ownership is checked again after every conflicting write.
|
||||
pub async fn update_manual_transition_job_record<F>(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
update: F,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord>
|
||||
where
|
||||
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
|
||||
{
|
||||
update_manual_transition_job_record_from(api, job_id, expected_lease_id, None, update).await
|
||||
}
|
||||
|
||||
async fn update_manual_transition_job_record_from<F>(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
mut current: Option<(ManualTransitionJobRecord, String)>,
|
||||
mut update: F,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord>
|
||||
where
|
||||
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
|
||||
{
|
||||
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
|
||||
let (mut record, etag) = match current.take() {
|
||||
Some(current) => current,
|
||||
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
|
||||
};
|
||||
if expected_lease_id.is_some_and(|lease_id| record.lease_id != lease_id) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if !update(&mut record) {
|
||||
return Ok(record);
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_manual_transition_job_before_first_cas(job_id).await;
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => return Ok(record),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub(crate) async fn save_manual_transition_worker_result_if_absent(
|
||||
api: Arc<ECStore>,
|
||||
record: &ManualTransitionWorkerResultRecord,
|
||||
@@ -1261,10 +1112,6 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_task_record(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1328,10 +1175,6 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_worker_result_stats(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1467,121 +1310,103 @@ async fn scan_manual_transition_worker_result_journal(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn reconcile_manual_transition_worker_results(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
reconcile_manual_transition_worker_results_inner(api, job_id, None, queue_snapshot, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_manual_transition_worker_results_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, false).await
|
||||
}
|
||||
|
||||
async fn reconcile_manual_transition_worker_results_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
mark_missing_results_unknown: bool,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let task_stats = match scan_manual_transition_task_journal(api.clone(), job_id).await? {
|
||||
ManualTransitionTaskJournal::Stats(stats) => stats,
|
||||
ManualTransitionTaskJournal::Corrupt(error) => {
|
||||
return mark_manual_transition_job_unknown_for_task_journal_error(
|
||||
api,
|
||||
job_id,
|
||||
expected_lease_id,
|
||||
error,
|
||||
queue_snapshot,
|
||||
)
|
||||
.await;
|
||||
return mark_manual_transition_job_unknown_for_task_journal_error(api, job_id, error, queue_snapshot).await;
|
||||
}
|
||||
};
|
||||
let stats = match scan_manual_transition_worker_result_journal(api.clone(), job_id).await? {
|
||||
ManualTransitionWorkerResultJournal::Stats(stats) => stats,
|
||||
ManualTransitionWorkerResultJournal::Corrupt(error) => {
|
||||
return mark_manual_transition_job_unknown_for_worker_result_journal_error(
|
||||
api,
|
||||
job_id,
|
||||
expected_lease_id,
|
||||
error,
|
||||
queue_snapshot,
|
||||
)
|
||||
.await;
|
||||
return mark_manual_transition_job_unknown_for_worker_result_journal_error(api, job_id, error, queue_snapshot).await;
|
||||
}
|
||||
};
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
let counts_changed = record.apply_worker_result_counts(
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
let changed = record.apply_worker_result_counts(
|
||||
stats.stats.completed,
|
||||
stats.stats.failed,
|
||||
&stats.stats.tier_failure_by_reason,
|
||||
task_stats.queued,
|
||||
queue_snapshot,
|
||||
);
|
||||
let became_unknown = mark_missing_results_unknown && record.mark_unknown_if_worker_results_lost(queue_snapshot);
|
||||
changed = counts_changed || became_unknown;
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if !changed {
|
||||
return Ok(record);
|
||||
if !changed {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(
|
||||
api.clone(),
|
||||
&record.scope_key,
|
||||
record.job_id,
|
||||
record.lease_id,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
Ok(record)
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
async fn mark_manual_transition_job_unknown_for_task_journal_error(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
error: String,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
changed = record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot);
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if changed && record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if !record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot) {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
|
||||
.await?;
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Ok(record)
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
async fn mark_manual_transition_job_unknown_for_worker_result_journal_error(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Option<Uuid>,
|
||||
error: String,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let mut changed = false;
|
||||
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
|
||||
changed = record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot);
|
||||
changed
|
||||
})
|
||||
.await?;
|
||||
if changed && record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if !record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot) {
|
||||
return Ok(record);
|
||||
}
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
|
||||
.await?;
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Ok(record)
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
@@ -1778,14 +1603,19 @@ async fn find_active_legacy_manual_transition_scope_conflict(
|
||||
}
|
||||
|
||||
pub async fn request_manual_transition_job_cancel(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
update_manual_transition_job_record(api, job_id, None, |record| {
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.is_terminal() || record.cancel_requested {
|
||||
return false;
|
||||
return Ok(record);
|
||||
}
|
||||
record.mark_cancel_requested();
|
||||
true
|
||||
})
|
||||
.await
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => return Ok(record),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
|
||||
pub async fn persist_manual_transition_job_progress(
|
||||
@@ -1794,39 +1624,10 @@ pub async fn persist_manual_transition_job_progress(
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
persist_manual_transition_job_progress_inner(api, job_id, current.0.lease_id, Some(current), report, queue_snapshot).await
|
||||
}
|
||||
|
||||
pub async fn persist_manual_transition_job_progress_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
persist_manual_transition_job_progress_inner(api, job_id, expected_lease_id, None, report, queue_snapshot).await
|
||||
}
|
||||
|
||||
async fn persist_manual_transition_job_progress_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
current: Option<(ManualTransitionJobRecord, String)>,
|
||||
report: &ManualTransitionRunReport,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let record = update_manual_transition_job_record_from(api.clone(), job_id, Some(expected_lease_id), current, |record| {
|
||||
if record.state != ManualTransitionJobState::Running {
|
||||
return false;
|
||||
}
|
||||
record.update_running_progress(report.clone(), queue_snapshot);
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
if record.state == ManualTransitionJobState::Running {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
record.update_running_progress(report.clone(), queue_snapshot);
|
||||
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -1860,58 +1661,25 @@ pub async fn renew_manual_transition_job_lease(
|
||||
job_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
renew_manual_transition_job_lease_inner(api, job_id, current.0.lease_id, Some(current), queue_snapshot).await
|
||||
}
|
||||
|
||||
pub async fn renew_manual_transition_job_lease_if_owned(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
renew_manual_transition_job_lease_inner(api, job_id, expected_lease_id, None, queue_snapshot).await
|
||||
}
|
||||
|
||||
async fn renew_manual_transition_job_lease_inner(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
expected_lease_id: Uuid,
|
||||
current: Option<(ManualTransitionJobRecord, String)>,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let (current, current_etag) = match current {
|
||||
Some(current) => current,
|
||||
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
|
||||
};
|
||||
if current.lease_id != expected_lease_id {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if current.state != ManualTransitionJobState::Running {
|
||||
return Ok(current);
|
||||
}
|
||||
if current.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
|
||||
return reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, true)
|
||||
.await;
|
||||
}
|
||||
let record = update_manual_transition_job_record_from(
|
||||
api.clone(),
|
||||
job_id,
|
||||
Some(expected_lease_id),
|
||||
Some((current, current_etag)),
|
||||
|record| {
|
||||
if record.state != ManualTransitionJobState::Running {
|
||||
return false;
|
||||
let (mut record, mut etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.state == ManualTransitionJobState::Running {
|
||||
if record.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
|
||||
record = reconcile_manual_transition_worker_results(api.clone(), job_id, queue_snapshot).await?;
|
||||
if record.is_terminal() || !record.report.worker_transition_pending() {
|
||||
return Ok(record);
|
||||
}
|
||||
(record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
}
|
||||
let became_terminal = record.mark_unknown_if_worker_results_lost(queue_snapshot);
|
||||
if !became_terminal {
|
||||
record.renew_lease(queue_snapshot);
|
||||
true
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else if record.state == ManualTransitionJobState::Running {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
|
||||
if became_terminal {
|
||||
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
@@ -1920,31 +1688,15 @@ async fn renew_manual_transition_scope_admission_from_job(
|
||||
api: Arc<ECStore>,
|
||||
record: &ManualTransitionJobRecord,
|
||||
) -> EcstoreResult<()> {
|
||||
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
|
||||
let (admission, admission_etag) =
|
||||
match load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await {
|
||||
Ok(admission) => admission,
|
||||
Err(Error::ConfigNotFound) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if admission.job_id != record.job_id || admission.lease_id != record.lease_id {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let mut renewed_admission = ManualTransitionScopeAdmission::from_job(record);
|
||||
renewed_admission.lease_expires_at_unix_nanos = renewed_admission
|
||||
.lease_expires_at_unix_nanos
|
||||
.max(admission.lease_expires_at_unix_nanos);
|
||||
renewed_admission.updated_at_unix_nanos = renewed_admission.updated_at_unix_nanos.max(admission.updated_at_unix_nanos);
|
||||
if renewed_admission == admission {
|
||||
return Ok(());
|
||||
}
|
||||
match save_manual_transition_scope_admission_if_current(api.clone(), &renewed_admission, &admission_etag).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
if let Ok((admission, admission_etag)) =
|
||||
load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await
|
||||
&& admission.job_id == record.job_id
|
||||
&& admission.lease_id == record.lease_id
|
||||
{
|
||||
let renewed_admission = ManualTransitionScopeAdmission::from_job(record);
|
||||
save_manual_transition_scope_admission_if_current(api, &renewed_admission, &admission_etag).await?;
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_manual_transition_scope_admission_if_current(
|
||||
@@ -2634,14 +2386,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_control_plane_failure_does_not_count_tier_failure() {
|
||||
fn manual_transition_job_record_failure_counts_tier_failure() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
|
||||
record.fail("missing tier");
|
||||
|
||||
assert_eq!(record.state, ManualTransitionJobState::Failed);
|
||||
assert_eq!(record.report.tier_failure, 0);
|
||||
assert_eq!(record.report.tier_failure, 1);
|
||||
assert_eq!(record.error.as_deref(), Some("missing tier"));
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
|
||||
@@ -15,35 +15,25 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||
obj.version_purge_status.is_pending()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||
}
|
||||
|
||||
@@ -14,10 +14,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||
}
|
||||
|
||||
@@ -331,10 +331,6 @@ where
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -148,10 +148,6 @@ struct RecoveryCursor {
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn recover_tier_free_versions(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
|
||||
@@ -80,10 +80,7 @@ impl LastDayTierStats {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m;
|
||||
|
||||
@@ -177,10 +177,9 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
struct ObjSweeper {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
bucket: String,
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
@@ -192,9 +191,9 @@ struct ObjSweeper {
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ObjSweeper {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||
Ok(Self {
|
||||
object: object.into(),
|
||||
@@ -203,20 +202,17 @@ impl ObjSweeper {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||
self.versioned = versioned;
|
||||
self.suspended = suspended;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id.clone(),
|
||||
@@ -230,7 +226,6 @@ impl ObjSweeper {
|
||||
opts
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||
self.transition_tier = info.tier;
|
||||
self.transition_status = info.status;
|
||||
@@ -271,7 +266,6 @@ impl ObjSweeper {
|
||||
None
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
@@ -391,10 +385,6 @@ impl ExpiryOp for Jentry {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||
if let Err(err) = &result
|
||||
@@ -405,10 +395,6 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
#[cfg(test)]
|
||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||
@@ -419,10 +405,6 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -503,10 +485,6 @@ pub enum RemoteTierDeleteOutcome {
|
||||
AlreadyRemoved,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
|
||||
@@ -50,16 +50,8 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionTransactionError {
|
||||
#[error("transition transaction already exists")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
AlreadyExists,
|
||||
#[error("transition transaction is not found")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
NotFound,
|
||||
#[error("transition transaction is corrupt: {0}")]
|
||||
Corrupt(&'static str),
|
||||
|
||||
@@ -50,75 +50,6 @@ use uuid::Uuid;
|
||||
|
||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
struct ConfigWriteLockProbeState {
|
||||
bucket: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub struct ConfigWriteLockProbe {
|
||||
state: Arc<ConfigWriteLockProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl ConfigWriteLockProbe {
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(ConfigWriteLockProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
assert!(
|
||||
!probes.iter().any(|current| current.bucket == state.bucket),
|
||||
"config write lock probe must be unique for a bucket"
|
||||
);
|
||||
probes.push(Arc::clone(&state));
|
||||
drop(probes);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub async fn wait_until_attempted(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("bucket config update should attempt the transaction lock");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for ConfigWriteLockProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison");
|
||||
probes.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_config_write_lock_attempt(bucket: &str) {
|
||||
let probe = CONFIG_WRITE_LOCK_PROBES
|
||||
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("config write lock probe mutex should not poison")
|
||||
.iter()
|
||||
.find(|probe| probe.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(probe) = probe {
|
||||
probe.arrived.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MetadataLoadMode {
|
||||
Initial,
|
||||
@@ -659,41 +590,6 @@ pub async fn update_under_transaction_lock(
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||
pub async fn delete_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||
}
|
||||
|
||||
pub async fn update_quota_if_incarnation(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||
sys.clone(),
|
||||
bucket,
|
||||
Some(expected_incarnation_id),
|
||||
))
|
||||
.await?;
|
||||
if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
|
||||
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
bucket: bucket.to_string(),
|
||||
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
@@ -808,14 +704,6 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||
}
|
||||
|
||||
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
||||
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -846,26 +734,7 @@ async fn acquire_transaction_lock_with_sys(
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
tokio::pin!(acquire);
|
||||
let mut notified = false;
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if !notified {
|
||||
notify_config_write_lock_attempt(bucket);
|
||||
notified = true;
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await?;
|
||||
Ok(guard)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
Ok(acquire.await?)
|
||||
Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||
@@ -893,10 +762,6 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1024,37 +889,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-read the quota configuration and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the bucket metadata
|
||||
/// transaction read lock.
|
||||
pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
match bucket_meta_sys
|
||||
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
.await?
|
||||
{
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
metadata.quota_config_updated_at,
|
||||
))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
}
|
||||
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1111,10 +945,6 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1628,7 +1458,6 @@ impl BucketMetadataSys {
|
||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||
/// instead of supplied up front. Loads through this system's own store so
|
||||
/// the read and the persisted write target the same instance.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
@@ -1733,7 +1562,6 @@ impl BucketMetadataSys {
|
||||
/// A miss is never published as an authoritative default, and a snapshot
|
||||
/// read before delete plus same-name recreation cannot replace the new
|
||||
/// generation.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
|
||||
@@ -136,7 +136,6 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
||||
|
||||
/// Check if an object has legal hold enabled.
|
||||
/// Returns true if legal hold is ON.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||
@@ -152,7 +151,6 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
||||
/// # Returns
|
||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||
/// * `false` if the object is not locked
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||
// Delete markers are never locked
|
||||
if is_delete_marker {
|
||||
|
||||
@@ -52,7 +52,6 @@ impl QuotaChecker {
|
||||
) -> Result<QuotaCheckResult, QuotaError> {
|
||||
let start_time = Instant::now();
|
||||
let quota_config = self.get_quota_config(bucket).await?;
|
||||
let uses_durable_reservations = quota_config.uses_durable_reservations();
|
||||
|
||||
// If no quota limit is set, allow operation
|
||||
let quota_limit = match quota_config.quota {
|
||||
@@ -68,7 +67,6 @@ impl QuotaChecker {
|
||||
quota_limit: None,
|
||||
operation_size,
|
||||
remaining: None,
|
||||
uses_durable_reservations,
|
||||
});
|
||||
}
|
||||
Some(q) => q,
|
||||
@@ -76,17 +74,14 @@ impl QuotaChecker {
|
||||
|
||||
let current_usage = self.get_real_time_usage(bucket).await?;
|
||||
|
||||
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
|
||||
let expected_usage = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
current_usage.saturating_add(admission_size)
|
||||
}
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
|
||||
QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
|
||||
};
|
||||
|
||||
let allowed = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
quota_config.check_operation_allowed(current_usage, admission_size)
|
||||
quota_config.check_operation_allowed(current_usage, operation_size)
|
||||
}
|
||||
QuotaOperation::DeleteObject => true,
|
||||
};
|
||||
@@ -110,7 +105,6 @@ impl QuotaChecker {
|
||||
quota_limit: Some(quota_limit),
|
||||
operation_size,
|
||||
remaining,
|
||||
uses_durable_reservations,
|
||||
};
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
@@ -164,26 +158,6 @@ impl QuotaChecker {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn set_durable_quota_config_if_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
quota: BucketQuota,
|
||||
expected_incarnation_id: uuid::Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime, QuotaError> {
|
||||
let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig {
|
||||
reason: format!("Failed to serialize quota config: {}", e),
|
||||
})?;
|
||||
let start_time = Instant::now();
|
||||
let updated_at =
|
||||
crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
|
||||
.await
|
||||
.map_err(QuotaError::StorageError)?;
|
||||
|
||||
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
async fn set_quota_config_for_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
@@ -381,7 +355,6 @@ mod tests {
|
||||
quota_limit: None,
|
||||
operation_size: 1024,
|
||||
remaining: None,
|
||||
uses_durable_reservations: false,
|
||||
};
|
||||
|
||||
assert!(result.allowed);
|
||||
@@ -405,13 +378,4 @@ mod tests {
|
||||
let allowed = quota.check_operation_allowed(512, 1024);
|
||||
assert!(!allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
|
||||
let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
|
||||
let durable = BucketQuota::new(Some(5));
|
||||
|
||||
assert!(!legacy.check_operation_allowed(4, 2));
|
||||
assert!(durable.uses_durable_reservations());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,98 +13,38 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod checker;
|
||||
pub(crate) mod reservation;
|
||||
|
||||
use crate::error::Result;
|
||||
use rustfs_config::{
|
||||
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
|
||||
QUOTA_NOT_FOUND_ERROR_CODE,
|
||||
};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub enum QuotaType {
|
||||
/// Hard quota accounting.
|
||||
/// Hard quota: reject immediately when exceeded
|
||||
#[default]
|
||||
#[serde(alias = "HARD", alias = "hard")]
|
||||
Hard,
|
||||
}
|
||||
|
||||
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||
|
||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
|
||||
pub struct BucketQuota {
|
||||
#[serde(default)]
|
||||
pub quota: Option<u64>,
|
||||
/// Defaults to Hard when missing.
|
||||
#[serde(default)]
|
||||
pub quota_type: QuotaType,
|
||||
/// Optional durable reservation protocol. The wire format gives older
|
||||
/// nodes a zero hard quota so a mixed-version fleet fails closed.
|
||||
pub reservation_protocol: Option<u32>,
|
||||
/// Timestamp when this quota configuration was set (for audit purposes)
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
pub created_at: Option<OffsetDateTime>,
|
||||
/// Accept updated_at for compatibility; not used.
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct BucketQuotaWire {
|
||||
#[serde(default)]
|
||||
quota: Option<u64>,
|
||||
#[serde(default)]
|
||||
quota_type: QuotaType,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_protocol: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
reservation_quota: Option<u64>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
created_at: Option<OffsetDateTime>,
|
||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl Serialize for BucketQuota {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let durable = self.uses_durable_reservations();
|
||||
BucketQuotaWire {
|
||||
quota: if durable { Some(0) } else { self.quota },
|
||||
quota_type: self.quota_type.clone(),
|
||||
reservation_protocol: self.reservation_protocol,
|
||||
reservation_quota: if durable { self.quota } else { None },
|
||||
created_at: self.created_at,
|
||||
updated_at: self.updated_at,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BucketQuota {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||
Some(
|
||||
wire.reservation_quota
|
||||
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
|
||||
)
|
||||
} else {
|
||||
wire.quota
|
||||
};
|
||||
Ok(Self {
|
||||
quota,
|
||||
quota_type: wire.quota_type,
|
||||
reservation_protocol: wire.reservation_protocol,
|
||||
created_at: wire.created_at,
|
||||
updated_at: wire.updated_at,
|
||||
})
|
||||
}
|
||||
pub updated_at: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
impl BucketQuota {
|
||||
@@ -123,7 +63,6 @@ impl BucketQuota {
|
||||
Self {
|
||||
quota,
|
||||
quota_type: QuotaType::Hard,
|
||||
reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
|
||||
created_at: Some(now),
|
||||
updated_at: None,
|
||||
}
|
||||
@@ -133,19 +72,7 @@ impl BucketQuota {
|
||||
self.quota
|
||||
}
|
||||
|
||||
pub fn uses_durable_reservations(&self) -> bool {
|
||||
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||
self.reservation_protocol
|
||||
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||
}
|
||||
|
||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||
if operation_size == 0 {
|
||||
return true;
|
||||
}
|
||||
if let Some(quota_limit) = self.quota {
|
||||
current_usage.saturating_add(operation_size) <= quota_limit
|
||||
} else {
|
||||
@@ -167,7 +94,6 @@ pub struct QuotaCheckResult {
|
||||
pub quota_limit: Option<u64>,
|
||||
pub operation_size: u64,
|
||||
pub remaining: Option<u64>,
|
||||
pub uses_durable_reservations: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -193,7 +119,6 @@ pub enum QuotaError {
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub struct QuotaErrorResponse {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: String,
|
||||
@@ -209,7 +134,6 @@ pub struct QuotaErrorResponse {
|
||||
}
|
||||
|
||||
impl QuotaErrorResponse {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||
match quota_error {
|
||||
QuotaError::QuotaExceeded { .. } => Self {
|
||||
@@ -286,57 +210,7 @@ mod tests {
|
||||
let buf = q.marshal_msg().expect("marshal");
|
||||
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
|
||||
assert_eq!(q.quota, restored.quota);
|
||||
assert_eq!(restored.quota_type, QuotaType::Hard);
|
||||
assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clearing_quota_keeps_the_legacy_compatible_type() {
|
||||
let quota = BucketQuota::new(None);
|
||||
|
||||
assert_eq!(quota.quota_type, QuotaType::Hard);
|
||||
assert_eq!(quota.reservation_protocol, None);
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_quota_makes_legacy_nodes_fail_closed() {
|
||||
let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
|
||||
let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
|
||||
assert!(quota.uses_durable_reservations());
|
||||
assert_eq!(quota.quota, Some(2048));
|
||||
|
||||
#[derive(Deserialize)]
|
||||
enum LegacyQuotaType {
|
||||
Hard,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
quota: Option<u64>,
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
.expect("legacy readers should ignore the reservation protocol field");
|
||||
assert_eq!(legacy.quota, Some(0));
|
||||
assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||
let quota: BucketQuota =
|
||||
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||
.expect("future protocol should remain parseable");
|
||||
|
||||
assert!(!quota.uses_durable_reservations());
|
||||
assert!(quota.has_unsupported_reservation_protocol());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reservation_protocol_v1_requires_reservation_quota() {
|
||||
let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
|
||||
.expect_err("v1 without its authoritative quota must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("reservation_quota is required"));
|
||||
assert_eq!(q.quota_type, restored.quota_type);
|
||||
}
|
||||
|
||||
/// unmarshal accepts format without quota_type
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,9 +11,9 @@ paths.
|
||||
| Module | Current role | Split blocker |
|
||||
|---|---|---|
|
||||
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local filemeta/tagging boundaries and S3 DTOs directly. |
|
||||
| `datatypes.rs` | ECStore compatibility re-export for resync status enums. | Re-exports `rustfs-replication` contracts while downstream facade consumers migrate. |
|
||||
| `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. |
|
||||
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
|
||||
| `replication_proxy.rs` | Proxy-target selection for GET/HEAD/Tagging reads of objects not yet replicated locally (MinIO `getProxyTargets` parity: anti-loop, version-suspended, and no-config empty branches). | Uses replication config lookup, rule matching, and target clients through local boundaries. |
|
||||
| `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. |
|
||||
| `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. |
|
||||
| `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
|
||||
@@ -117,12 +117,9 @@ Target end state:
|
||||
their file names — so batch-merging them beforehand is explicitly rejected:
|
||||
it forces synchronized guard-script/mod/import churn with zero functional
|
||||
gain;
|
||||
- `datatypes.rs` retired early (its sanctioned exception): it was a pure
|
||||
relay (`boundary -> datatypes -> mod.rs`), so the facade now re-exports
|
||||
`ResyncStatusType` from the resync boundary directly and the relay file is
|
||||
deleted. Note the original retirement wording ("consumers import through
|
||||
`rustfs-replication` directly") conflicted with Migration Rule #15 —
|
||||
consumers stay behind the ECStore facade; only the relay hop dissolves.
|
||||
- the only module that can retire early is `datatypes.rs`: delete it once its
|
||||
facade consumers import the resync status enums through `rustfs-replication`
|
||||
directly.
|
||||
|
||||
## Milestones
|
||||
|
||||
@@ -130,9 +127,9 @@ Target end state:
|
||||
|---|---|---|
|
||||
| M0 | Record the completion criteria and end state (this section). | Done |
|
||||
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Done — moved the pure decision helpers with their unit tests: `resync_status_duration` (resync), `resync_existing_delete_replication_info` / `replicate_delete_outcome` / `target_delete_version_id` / `delete_marker_purge_version_id` / `delete_marker_purge_mrf_entry` (delete), `version_identity_drifted` / `is_replication_target_offline_error` / the SSE-C passthrough gate family incl. `SsecPassthroughCapability` (object; `ssec_passthrough_evidence_present` was param-demoted to the echoed customer-algorithm string, ECStore keeps the `HeadObjectOutput` adapter). ECStore imports them through the resync/object-decision/target boundaries; `bucket_target_sys` keeps only the verdict cache + TTL and re-exports the capability enum. Not moved (signatures carry ECStore or aws-sdk types): `verify_resync_head_result`, `resync_target_error_detail`, the `SdkError` classifiers (`has_raw_status`, `is_version_id_format_mismatch`), the `replicate_all_*` option/info builders, and `bounded_resync_max_jobs` (itself a pure clamp, but it forms one local configuration unit with the env-reading `configured_resync_max_jobs` and its ECStore-local constants — moving the clamp alone has negative value). |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
|
||||
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries. | Pending (`datatypes.rs` already retired early alongside M2) |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
|
||||
|
||||
The original first code-bearing step (narrow `ReplicationEventSink` /
|
||||
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod datatypes;
|
||||
mod replication_bandwidth_boundary;
|
||||
mod replication_config_boundary;
|
||||
mod replication_config_store;
|
||||
@@ -28,7 +29,6 @@ mod replication_object_bridge;
|
||||
mod replication_object_config;
|
||||
mod replication_object_decision_boundary;
|
||||
pub(crate) mod replication_pool;
|
||||
mod replication_proxy;
|
||||
mod replication_queue_boundary;
|
||||
mod replication_resync_boundary;
|
||||
mod replication_resyncer;
|
||||
@@ -43,6 +43,7 @@ pub(crate) mod replication_timing;
|
||||
mod replication_versioning_boundary;
|
||||
mod runtime_boundary;
|
||||
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
@@ -59,7 +60,7 @@ pub use replication_filemeta_boundary::{
|
||||
pub(crate) use replication_filemeta_boundary::{
|
||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
};
|
||||
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
@@ -73,16 +74,13 @@ pub use replication_pool::{
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_proxy::get_proxy_targets;
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
};
|
||||
pub use replication_resync_boundary::ResyncStatusType;
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -37,10 +37,6 @@ impl ReplicationConfigStore {
|
||||
com::read_config_limited(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
|
||||
@@ -12,11 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_filemeta::NULL_VERSION_ID;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision,
|
||||
replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
||||
parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
pub use rustfs_replication::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
|
||||
@@ -24,27 +24,15 @@ use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(
|
||||
config: &ReplicationLifecycleConfig,
|
||||
object_name: &str,
|
||||
@@ -57,10 +45,6 @@ impl ReplicationLifecycleBridge {
|
||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: &ObjectToDelete,
|
||||
@@ -70,10 +54,6 @@ impl ReplicationLifecycleBridge {
|
||||
check_replicate_delete(bucket, object, source, opts, None).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||
let pending_status = decision.pending_status();
|
||||
ReplicationState {
|
||||
|
||||
@@ -19,33 +19,17 @@ use time::OffsetDateTime;
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::msgp_decode;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) struct ReplicationMsgpCodec;
|
||||
|
||||
impl ReplicationMsgpCodec {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||
msgp_decode::read_msgp_ext8_time(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||
msgp_decode::skip_msgp_value(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||
msgp_decode::write_msgp_time(wr, time)
|
||||
}
|
||||
|
||||
@@ -77,10 +77,6 @@ impl ReplicationObjectBridge {
|
||||
load_delete_request_config_in(ctx, bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn delete_config_snapshot_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -231,10 +231,6 @@ pub(crate) async fn load_delete_replication_config(
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn load_delete_replication_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -18,10 +18,9 @@ pub use rustfs_replication::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject,
|
||||
delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object,
|
||||
should_retry_delete_marker_purge,
|
||||
};
|
||||
|
||||
@@ -217,10 +217,6 @@ impl DurableMrfBacklogTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||
let mut tracker = DurableMrfBacklogTracker {
|
||||
available: true,
|
||||
@@ -667,368 +663,6 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Some(guard),
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and decodes the on-disk MRF recovery file.
|
||||
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return None;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match decode_mrf_file(&data) {
|
||||
Ok(v) => Some(v),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(storage, &data).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays one MRF recovery entry by operation kind.
|
||||
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||
}
|
||||
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||
/// stale force-delete generations are skipped, and plain deletes are
|
||||
/// reconstructed as heal deletes.
|
||||
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let operation_id = entry.force_delete_id?;
|
||||
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
Some(ReplicationQueueAdmission::Skipped)
|
||||
} else {
|
||||
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
/// get_object_info here because the delete-marker or version may
|
||||
/// already be absent from the local store — that is expected.
|
||||
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The MRF entry does not persist the replication decision and the
|
||||
/// source object is gone, so re-derive the decision from the live
|
||||
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
/// it on the reconstructed delete. Without this the decision string
|
||||
/// is empty and the delete replicates to zero targets — a silent
|
||||
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
async fn resolve_mrf_delete_replicate_decision(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
versioned: bool,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicateDecision> {
|
||||
if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => None,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => Some(dsc),
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||
/// replication decision.
|
||||
fn reconstructed_heal_delete_info(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a metadata-kind MRF entry against the live source object.
|
||||
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||
/// admitted targets, carrying over the entry's retry count.
|
||||
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
roi
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
entries: &[MrfReplicateEntry],
|
||||
retry_entries: &[MrfReplicateEntry],
|
||||
) -> Vec<MrfReplicateEntry> {
|
||||
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.to_vec()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1078,10 +712,6 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
|
||||
// MRF worker lifecycle
|
||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
mrf_stop_tx: Sender<()>,
|
||||
|
||||
// Worker size tracking
|
||||
@@ -1310,10 +940,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Resizes worker priority and counts
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn resize_worker_priority(
|
||||
&self,
|
||||
pri: ReplicationPriority,
|
||||
@@ -1554,10 +1180,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Queues an MRF save operation
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
@@ -1583,12 +1205,71 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let recovery_guard = match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries = match decode_mrf_file(&data) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(&storage, &data).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
@@ -1597,8 +1278,187 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
let admission = match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let Some(operation_id) = entry.force_delete_id else {
|
||||
continue;
|
||||
};
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
ReplicationQueueAdmission::Skipped
|
||||
} else {
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
//
|
||||
// The MRF entry does not persist the replication decision and the
|
||||
// source object is gone, so re-derive the decision from the live
|
||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
// it on the reconstructed delete. Without this the decision string
|
||||
// is empty and the delete replicates to zero targets — a silent
|
||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => continue,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => dsc,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
||||
};
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Metadata => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
@@ -1608,7 +1468,29 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
@@ -1769,10 +1651,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling regular replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1786,10 +1664,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1804,10 +1678,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_mrf_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1821,10 +1691,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Delete resync metadata from replication resync state in memory
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||
let mut status_map = self.resyncer.status_map.write().await;
|
||||
status_map.remove(bucket);
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Proxy-target selection for reads of objects not yet replicated locally
|
||||
//! (MinIO `getProxyTargets`, bucket-replication.go).
|
||||
//!
|
||||
//! During the active-active replication lag window a GET/HEAD/Tagging request
|
||||
//! for an object the local site does not have yet may be served by proxying to
|
||||
//! a replication target. This module only *selects* the candidate targets; the
|
||||
//! request-path callers perform the remote calls and response translation.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_object_config::get_replication_config;
|
||||
use super::replication_storage_boundary::ObjectOptions;
|
||||
use super::replication_target_boundary::{ReplicationTargetStore, TargetClient};
|
||||
|
||||
/// Returns the replication-target clients eligible to serve a proxied read of
|
||||
/// `bucket/object`, in rule order. Mirrors MinIO's `getProxyTargets`:
|
||||
///
|
||||
/// - the `source-proxy-request` header family was present at all
|
||||
/// (`opts.proxy_request` / `opts.proxy_header_set`, MinIO `ProxyRequest` /
|
||||
/// `ProxyHeaderSet`) -> empty. "true" is the anti-loop marker of an
|
||||
/// already-proxied client read; "false" is what a peer's replication
|
||||
/// worker sends on convergence HEADs so the receiver answers locally —
|
||||
/// proxying that miss back would echo the source object and fake
|
||||
/// convergence, permanently skipping replication;
|
||||
/// - the bucket's versioning is suspended for the object -> empty;
|
||||
/// - no replication configuration / no matching rule -> empty;
|
||||
/// - otherwise every distinct target ARN whose rules match the object,
|
||||
/// resolved through the bucket target system, skipping targets that opted
|
||||
/// out of proxying (`disable_proxy`).
|
||||
pub async fn get_proxy_targets(bucket: &str, object: &str, opts: &ObjectOptions) -> Vec<Arc<TargetClient>> {
|
||||
if opts.proxy_request || opts.proxy_header_set {
|
||||
return Vec::new();
|
||||
}
|
||||
if opts.version_suspended {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let cfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(cfg)) => cfg,
|
||||
Ok(None) => return Vec::new(),
|
||||
Err(err) => {
|
||||
debug!(bucket, object, error = %err, "read proxy: failed to load replication config; not proxying");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let arns = cfg.filter_target_arns(&ObjectOpts {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut targets = Vec::with_capacity(arns.len());
|
||||
for arn in arns {
|
||||
let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
debug!(bucket, object, arn, "read proxy: no client for replication target ARN");
|
||||
continue;
|
||||
};
|
||||
if client.disable_proxy {
|
||||
continue;
|
||||
}
|
||||
targets.push(client);
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn opts() -> ObjectOptions {
|
||||
ObjectOptions::default()
|
||||
}
|
||||
|
||||
/// Anti-loop: a request that was already proxied by a peer must never be
|
||||
/// proxied onward, regardless of replication configuration.
|
||||
#[tokio::test]
|
||||
async fn proxy_request_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_request: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// MinIO `ProxyHeaderSet` parity: the header family being present at all
|
||||
/// disables proxying, even with the value "false" — that is what a
|
||||
/// peer's replication worker sends on convergence HEADs.
|
||||
#[tokio::test]
|
||||
async fn proxy_header_set_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_header_set: true,
|
||||
proxy_request: false,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// Suspended versioning disables proxying (MinIO parity): the local null
|
||||
/// version is authoritative and a remote read could resurrect data.
|
||||
#[tokio::test]
|
||||
async fn version_suspended_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// A bucket without replication configuration has nothing to proxy to.
|
||||
/// (No metadata system is running in unit tests, so the config lookup
|
||||
/// resolves to "no configuration" — the same empty-result contract.)
|
||||
#[tokio::test]
|
||||
async fn missing_replication_config_yields_no_targets() {
|
||||
let targets = get_proxy_targets("bucket-without-replication", "object", &opts()).await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -15,42 +15,17 @@
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
use super::replication_filemeta_boundary::MrfReplicateEntry;
|
||||
|
||||
/// Kept test-only: the runtime consumer was the worker HEAD's fake proxy
|
||||
/// counting (removed in backlog#1675 P1-5); the resyncer tests still pin the
|
||||
/// classifier's semantics for the real client read-proxy failure accounting.
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::should_count_head_proxy_failure;
|
||||
pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus};
|
||||
pub(crate) use rustfs_replication::{
|
||||
is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
|
||||
should_auto_resume_resync,
|
||||
is_version_id_mismatch, resync_state_accepts_update, sanitize_resync_error_detail, should_auto_resume_resync,
|
||||
should_count_head_proxy_failure,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||
|
||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -340,10 +340,6 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Site replication update replica statistics
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn sr_update_replica_stat(&self, size: i64) {
|
||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -708,12 +704,6 @@ impl ReplicationStats {
|
||||
} else {
|
||||
BucketReplicationStats::new()
|
||||
};
|
||||
// Stamp the serializable failure windows from the live samples: the
|
||||
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
||||
// is what cluster aggregation and the metrics endpoints see.
|
||||
for stat in replication_stats.stats.values_mut() {
|
||||
stat.fail_stats.refresh_windows();
|
||||
}
|
||||
let uptime = if cache.contains_key(bucket) {
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
@@ -1161,31 +1151,6 @@ mod tests {
|
||||
assert!(all.contains_key("proxy-only-bucket"));
|
||||
}
|
||||
|
||||
/// Pins the read-proxy metric contract (backlog#1675 P1-5): the API
|
||||
/// strings the GET/HEAD/Tagging proxy paths record map onto the
|
||||
/// get/head/tagging totals, and only unexpected failures raise the
|
||||
/// failed counters.
|
||||
#[tokio::test]
|
||||
async fn test_proxy_stats_map_read_proxy_apis_to_totals() {
|
||||
let stats = ReplicationStats::new();
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", true).await;
|
||||
stats.inc_proxy("proxy-bucket", "HeadObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "PutObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "DeleteObjectTagging", true).await;
|
||||
|
||||
let metric = stats.get_proxy_stats("proxy-bucket").await;
|
||||
assert_eq!(metric.get_total, 2);
|
||||
assert_eq!(metric.get_failed, 1);
|
||||
assert_eq!(metric.head_total, 1);
|
||||
assert_eq!(metric.head_failed, 0);
|
||||
assert_eq!(metric.get_tag_total, 1);
|
||||
assert_eq!(metric.put_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_failed, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_calculate_bucket_replication_stats_merges_resync_metrics() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||
SRMetricsSummary, XferStats,
|
||||
};
|
||||
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
||||
// can project the internal stats onto the minio-go response shapes through
|
||||
// the storage_api facade chain.
|
||||
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||
|
||||
@@ -27,24 +27,18 @@ use rustfs_utils::http::{
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
||||
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
||||
ssec_replication_transport_header,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
pub use rustfs_replication::SsecPassthroughCapability;
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
};
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -69,8 +63,6 @@ static STANDARD_HEADERS: &[&str] = &[
|
||||
];
|
||||
|
||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||
pub(crate) const ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED: &str = "replication target does not support SSE-C passthrough: the replica would lose its decryption material \
|
||||
(run ?replication-check to re-probe)";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplicationSourceEncryption {
|
||||
@@ -127,38 +119,10 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
||||
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
||||
let key = key.as_bytes();
|
||||
key.len() == prefix.len() + suffix.len()
|
||||
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
||||
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
||||
}
|
||||
|
||||
[
|
||||
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| {
|
||||
["x-rustfs-", "x-minio-"]
|
||||
.iter()
|
||||
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||
/// customer-algorithm header and let the crate-owned policy decide.
|
||||
pub(crate) fn ssec_passthrough_evidence_present(head: &HeadObjectOutput) -> bool {
|
||||
rustfs_replication::ssec_passthrough_evidence_present(head.sse_customer_algorithm.as_deref())
|
||||
}
|
||||
|
||||
pub(crate) struct ReplicationTargetStore;
|
||||
|
||||
impl ReplicationTargetStore {
|
||||
@@ -178,17 +142,6 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||
BucketTargetSys::get()
|
||||
.record_ssec_passthrough_capability(arn, capability)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -223,11 +176,6 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_legacy_source_replication_timestamp_key(key) {
|
||||
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_internal_key(key) || is_standard_header(key) {
|
||||
continue;
|
||||
}
|
||||
@@ -311,23 +259,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
|
||||
if !tags.is_empty() {
|
||||
put_options.user_tags = tags;
|
||||
put_options.internal.tagging_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Load the stored tagging timestamp independently of whether any tags
|
||||
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||
// With no stored key, fall back to mod_time only while tags exist
|
||||
// (MinIO parity); a tagless object without the key was never tagged and
|
||||
// keeps the epoch default (no header).
|
||||
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||
{
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else if !put_options.user_tags.is_empty() {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
let metadata = &*object_info.user_defined;
|
||||
|
||||
@@ -343,15 +283,13 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
put_options.cache_control = cache_control.to_string();
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||
if !retain_until_date.is_empty() {
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
}
|
||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||
put_options.internal.retention_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
@@ -756,110 +694,6 @@ mod tests {
|
||||
assert!(options.internal.replication_request);
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||
/// still be loaded (and therefore sent) so the replica can order the
|
||||
/// deletion against concurrent tag edits.
|
||||
#[test]
|
||||
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
assert!(options.user_tags.is_empty());
|
||||
assert_eq!(
|
||||
options.internal.tagging_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||
"the stored tagging timestamp must load independently of remaining tags"
|
||||
);
|
||||
|
||||
// A tagless object without the stored key was never tagged: the epoch
|
||||
// default keeps the header unsent.
|
||||
let untagged = ObjectInfo {
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
||||
let legacy_keys = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(
|
||||
legacy_keys
|
||||
.iter()
|
||||
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
for legacy_key in legacy_keys {
|
||||
assert!(!options.user_metadata.contains_key(legacy_key));
|
||||
assert_eq!(
|
||||
options
|
||||
.user_metadata
|
||||
.get(&format!("x-amz-meta-{legacy_key}"))
|
||||
.map(String::as_str),
|
||||
Some("2099-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
||||
let mut metadata = HashMap::from([
|
||||
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
||||
]);
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
||||
|
||||
assert!(options.mode.is_none());
|
||||
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(
|
||||
options.internal.retention_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
||||
);
|
||||
let headers = options.header();
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
||||
Some("2026-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||
@@ -922,27 +756,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Pins the HeadObjectOutput field extraction feeding the crate-owned
|
||||
/// evidence judgment (the gate/evidence policy matrix itself is pinned in
|
||||
/// `rustfs-replication`'s object tests).
|
||||
#[test]
|
||||
fn ssec_passthrough_evidence_requires_customer_algorithm_echo() {
|
||||
let with_evidence = HeadObjectOutput::builder().sse_customer_algorithm("AES256").build();
|
||||
assert!(ssec_passthrough_evidence_present(&with_evidence));
|
||||
|
||||
let empty_algorithm = HeadObjectOutput::builder().sse_customer_algorithm("").build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&empty_algorithm),
|
||||
"an empty echo is not evidence of preserved SSE-C material"
|
||||
);
|
||||
|
||||
let without_evidence = HeadObjectOutput::builder().e_tag("\"abc\"").content_length(8).build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&without_evidence),
|
||||
"a plain HEAD response must classify the target as having dropped the material"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_adds_ssec_checksum_metadata() {
|
||||
let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||
|
||||
@@ -40,14 +40,7 @@ impl ARN {
|
||||
|
||||
impl Display for ARN {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
// The `minio` partition is deliberate: madmin-go's ParseARN
|
||||
// hard-rejects any other partition, so native mc/madmin tooling can
|
||||
// only decode remote-target ARNs minted in this form (backlog#1675
|
||||
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
||||
// readable via the FromStr whitelist below; runtime matching between
|
||||
// targets and replication rules is by full-string equality, so mixed
|
||||
// partitions coexist safely.
|
||||
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,12 +48,7 @@ impl FromStr for ARN {
|
||||
type Err = std::io::Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
||||
// from_str(...).unwrap_or_default()` below never fails, so this is
|
||||
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
||||
// the legacy partition and must stay accepted forever (persisted
|
||||
// bucket-targets.json / replication configs from older releases).
|
||||
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
||||
if !s.starts_with("arn:rustfs:") {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||
}
|
||||
|
||||
@@ -113,50 +101,14 @@ mod tests {
|
||||
}
|
||||
|
||||
/// RustFS commonly generates ARNs with an empty region:
|
||||
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||
#[test]
|
||||
fn from_str_handles_empty_region_segment() {
|
||||
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
||||
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
||||
/// native mc/madmin tooling cannot decode remote-target listings.
|
||||
#[test]
|
||||
fn display_emits_minio_partition() {
|
||||
let arn = ARN::new(
|
||||
BucketTargetType::ReplicationService,
|
||||
"depl-123".to_string(),
|
||||
String::new(),
|
||||
"bucket-a".to_string(),
|
||||
);
|
||||
|
||||
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
||||
}
|
||||
|
||||
/// Persisted bucket-targets.json files from older RustFS releases carry
|
||||
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
||||
#[test]
|
||||
fn from_str_accepts_legacy_rustfs_partition() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "us-east-1");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
||||
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
||||
/// would otherwise parse as `type=None`.
|
||||
#[test]
|
||||
fn from_str_rejects_unknown_partition() {
|
||||
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
||||
assert!(ARN::from_str("not-an-arn").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,10 +59,6 @@ impl fmt::Debug for Credentials {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub enum ServiceType {
|
||||
#[default]
|
||||
Replication,
|
||||
|
||||
@@ -73,6 +73,23 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
||||
if object_name.len() > 1024 {
|
||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
||||
}
|
||||
if !object_name.is_ascii() {
|
||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
||||
if object_name.trim().is_empty() {
|
||||
return Err(Error::other("Object name cannot be empty"));
|
||||
}
|
||||
check_valid_object_name_prefix(object_name)
|
||||
}
|
||||
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
@@ -83,10 +100,6 @@ where
|
||||
Ok(ans)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
@@ -173,27 +186,15 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||
/// accept the full S3 key character set.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||
|
||||
/// Client-facing reason for path segments Windows can store but not address
|
||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||
|
||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||
@@ -203,10 +204,6 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||
object.split(['/', '\\']).any(|segment| {
|
||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||
|
||||
@@ -90,10 +90,6 @@ impl BucketVersioningSys {
|
||||
/// caller's own instance context so a second in-process store never
|
||||
/// answers with the first instance's versioning state; falls back to the
|
||||
/// ambient system when the instance cell is not initialized.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||
return Ok(VersioningConfiguration::default());
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||
use std::{
|
||||
@@ -656,7 +655,6 @@ async fn list_path_raw_inner(
|
||||
errs.push(None);
|
||||
}
|
||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -678,21 +676,6 @@ async fn list_path_raw_inner(
|
||||
let mut has_err = 0;
|
||||
let mut agree = 0;
|
||||
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cancel = &revjob_rx;
|
||||
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
||||
});
|
||||
for (i, outcome) in join_all(concurrent_peeks).await {
|
||||
peek_outcomes[i] = Some(outcome);
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
has_err += 1;
|
||||
@@ -702,10 +685,7 @@ async fn list_path_raw_inner(
|
||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||
entry
|
||||
} else {
|
||||
let Some(outcome) = peek_outcomes[i].take() else {
|
||||
return Err(DiskError::Unexpected);
|
||||
};
|
||||
match outcome {
|
||||
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
||||
PeekOutcome::Ready(res) => {
|
||||
if let Some(entry) = res {
|
||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||
@@ -1315,36 +1295,6 @@ mod tests {
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
||||
let peek_timeout = Duration::from_millis(20);
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None, None, None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
TestReaderBehavior::Stall,
|
||||
],
|
||||
peek_timeout: Some(peek_timeout),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("all stalled readers should fail the listing");
|
||||
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
assert_eq!(
|
||||
started.elapsed(),
|
||||
peek_timeout,
|
||||
"reader deadlines must overlap instead of accumulating once per disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||
let entry = MetaCacheEntry {
|
||||
|
||||
@@ -229,6 +229,17 @@ pub fn http_resp_to_error_response(
|
||||
err_resp
|
||||
}
|
||||
|
||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
||||
.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||
@@ -284,6 +295,16 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
||||
message: message.to_string(),
|
||||
request_id: "rustfs".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GetRequest {
|
||||
pub buffer: Vec<u8>,
|
||||
pub offset: i64,
|
||||
@@ -106,12 +107,11 @@ pub struct GetRequest {
|
||||
pub setting_object_info: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct GetResponse {
|
||||
pub size: i64,
|
||||
//pub error: error,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub did_read: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub object_info: ObjectInfo,
|
||||
}
|
||||
|
||||
@@ -135,10 +135,6 @@ impl Object {
|
||||
Self { ..Default::default() }
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||
let _ = request.did_offset_change;
|
||||
let _ = request.offset;
|
||||
@@ -154,20 +150,12 @@ impl Object {
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||
self.curr_offset += bytes_read;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||
let mut read_req = GetRequest {
|
||||
is_read_op: true,
|
||||
@@ -192,10 +180,6 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let _ = self.do_get_request(&GetRequest {
|
||||
@@ -208,10 +192,6 @@ impl Object {
|
||||
Ok(self.object_info.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||
self.curr_offset = offset;
|
||||
|
||||
@@ -239,10 +219,6 @@ impl Object {
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let seek_req = GetRequest {
|
||||
@@ -277,10 +253,6 @@ impl Object {
|
||||
Ok(self.curr_offset)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||
self.is_closed = true;
|
||||
Ok(())
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
@@ -28,6 +27,7 @@ use tracing::warn;
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedGetOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub is_replication_ready_for_delete_marker: bool,
|
||||
@@ -77,7 +77,7 @@ impl GetObjectOptions {
|
||||
}
|
||||
}
|
||||
if self.checksum {
|
||||
headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
|
||||
headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
@@ -360,6 +360,7 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
with_versions: bool,
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::client::{
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
||||
credentials::SignatureType,
|
||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||
@@ -149,7 +149,6 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||
@@ -260,7 +259,6 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -30,6 +30,10 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||
let unknown_size;
|
||||
let mut object_size = object_size;
|
||||
|
||||
@@ -81,6 +81,18 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UploadedPartRes {
|
||||
pub error: std::io::Error,
|
||||
pub part_num: i64,
|
||||
pub size: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
pub struct UploadPartReq {
|
||||
pub part_num: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object_multipart_stream(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -55,6 +55,7 @@ pub struct RemoveBucketOptions {
|
||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub replication_status: ReplicationStatus,
|
||||
@@ -464,10 +465,10 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct RemoveObjectError {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object_name: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
version_id: String,
|
||||
err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
@@ -29,6 +29,10 @@ use crate::client::utils::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
pub struct ListAllMyBucketsResult {
|
||||
pub owner: Owner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CommonPrefix {
|
||||
pub prefix: String,
|
||||
@@ -85,10 +89,6 @@ pub struct ListVersionsResult {
|
||||
pub next_version_id_marker: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListBucketResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
contents: Vec<transition_api::ObjectInfo>,
|
||||
@@ -102,10 +102,6 @@ pub struct ListBucketResult {
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct ListMultipartUploadsResult {
|
||||
bucket: String,
|
||||
key_marker: String,
|
||||
@@ -121,15 +117,16 @@ pub struct ListMultipartUploadsResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||
)]
|
||||
pub struct Initiator {
|
||||
id: String,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
pub struct CopyObjectResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectPart {
|
||||
pub etag: String,
|
||||
@@ -263,7 +260,6 @@ pub struct CompletePart {
|
||||
}
|
||||
|
||||
impl CompletePart {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||
match t {
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
@@ -288,6 +284,11 @@ impl CompletePart {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CopyObjectPartResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
#[serde(rename = "CompleteMultipartUpload")]
|
||||
pub struct CompleteMultipartUpload {
|
||||
@@ -356,10 +357,10 @@ impl CompleteMultipartUpload {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
||||
)]
|
||||
pub struct CreateBucketConfiguration {
|
||||
pub location: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteObject {
|
||||
//api has
|
||||
@@ -367,6 +368,21 @@ pub struct DeleteObject {
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
pub struct DeletedObject {
|
||||
//s3s has
|
||||
pub key: String,
|
||||
pub version_id: String,
|
||||
pub deletemarker: bool,
|
||||
pub deletemarker_version_id: String,
|
||||
}
|
||||
|
||||
pub struct NonDeletedObject {
|
||||
pub key: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteMultiObjects {
|
||||
pub quiet: bool,
|
||||
@@ -386,7 +402,6 @@ impl DeleteMultiObjects {
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WireDeleteObject {
|
||||
@@ -421,3 +436,8 @@ impl DeleteMultiObjects {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DeleteMultiObjectsResult {
|
||||
pub deleted_objects: Vec<DeletedObject>,
|
||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
||||
}
|
||||
|
||||
@@ -27,24 +27,12 @@ use crate::client::utils::base64_decode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
use crate::{disk::DiskAPI, object_api::GetObjectReader};
|
||||
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
||||
// in rustfs-utils' headers module.
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
||||
use s3s::header::{
|
||||
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
|
||||
};
|
||||
|
||||
use enumset::{EnumSet, EnumSetType, enum_set};
|
||||
|
||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
||||
/// this enum is the MinIO-port client's wire vocabulary and stops at the
|
||||
/// standard S3 set (CRC64NVME is its newest member; the RustFS extensions do
|
||||
/// not exist on this client path). The streaming-hash registry lives in
|
||||
/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs) and
|
||||
/// the on-disk xl.meta bitset in `rustfs_rio::ChecksumType`
|
||||
/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an
|
||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
||||
/// each other.
|
||||
#[derive(Debug, EnumSetType, Default)]
|
||||
#[enumset(repr = "u8")]
|
||||
pub enum ChecksumMode {
|
||||
@@ -69,6 +57,8 @@ lazy_static! {
|
||||
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
|
||||
}
|
||||
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
|
||||
|
||||
impl ChecksumMode {
|
||||
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
|
||||
|
||||
@@ -365,15 +355,11 @@ mod tests {
|
||||
pub struct Checksum {
|
||||
checksum_type: ChecksumMode,
|
||||
r: Vec<u8>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
||||
)]
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl Checksum {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Checksum {
|
||||
@@ -385,7 +371,7 @@ impl Checksum {
|
||||
Checksum::default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||
let b = match base64_decode(s.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
@@ -412,7 +398,7 @@ impl Checksum {
|
||||
base64_encode(&self.r)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn raw(&self) -> Option<Vec<u8>> {
|
||||
if !self.is_set() {
|
||||
return None;
|
||||
|
||||
@@ -32,5 +32,8 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||
|
||||
pub const TOTAL_WORKERS: i64 = 4;
|
||||
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
@@ -67,10 +67,6 @@ impl<P: Provider + Default> Credentials<P> {
|
||||
Ok(self.creds.clone())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn expire(&mut self) {
|
||||
self.force_refresh = true;
|
||||
}
|
||||
@@ -137,10 +133,6 @@ impl Provider for Static {
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct STSError {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub r#type: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
@@ -149,10 +141,6 @@ pub struct STSError {
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub struct ErrorResponse {
|
||||
pub sts_error: STSError,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||
)]
|
||||
pub request_id: String,
|
||||
}
|
||||
|
||||
@@ -170,3 +158,22 @@ impl ErrorResponse {
|
||||
return self.sts_error.message.clone();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
match std::str::from_utf8(body) {
|
||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
let body = body_reader.to_vec();
|
||||
let parsed = xml_decoder(&body)?;
|
||||
Ok((body, parsed))
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_error_response;
|
||||
|
||||
@@ -37,17 +37,16 @@ pub struct PutObjReader {
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(reader: HashReader) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
self.reader = enc_reader;
|
||||
|
||||
@@ -78,6 +77,39 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
|
||||
})
|
||||
}
|
||||
|
||||
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
||||
let mut skip_length: i64 = 0;
|
||||
let mut cumulative_actual_size: i64 = 0;
|
||||
let mut first_part_idx: i64 = 0;
|
||||
let mut compressed_offset: i64 = 0;
|
||||
let mut part_skip: i64 = 0;
|
||||
let mut decrypt_skip: i64 = 0;
|
||||
let mut seq_num: u64 = 0;
|
||||
for (i, part) in oi.parts.iter().enumerate() {
|
||||
cumulative_actual_size += part.actual_size as i64;
|
||||
if cumulative_actual_size <= offset {
|
||||
compressed_offset += part.size as i64;
|
||||
} else {
|
||||
first_part_idx = i as i64;
|
||||
skip_length = cumulative_actual_size - part.actual_size as i64;
|
||||
break;
|
||||
}
|
||||
}
|
||||
skip_length = offset - skip_length;
|
||||
|
||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
||||
if skip_length > 0
|
||||
&& parts.len() > first_part_idx as usize
|
||||
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
||||
{
|
||||
let _ = part_skip;
|
||||
let _ = decrypt_skip;
|
||||
let _ = seq_num;
|
||||
}
|
||||
|
||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
||||
}
|
||||
|
||||
pub fn new_getobjectreader<'a>(
|
||||
rs: &Option<HTTPRangeSpec>,
|
||||
oi: &'a ObjectInfo,
|
||||
|
||||
@@ -23,7 +23,6 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
|
||||
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||
pub(crate) enum BucketVersioningState {
|
||||
Unknown,
|
||||
Disabled,
|
||||
@@ -48,7 +47,6 @@ impl RemoteVersion {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||
match self {
|
||||
Self::Unknown => Err(Error::new(
|
||||
|
||||
@@ -54,10 +54,6 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::{
|
||||
http::headers::{
|
||||
AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1,
|
||||
AMZ_CHECKSUM_SHA256,
|
||||
},
|
||||
net::get_endpoint_url,
|
||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||
};
|
||||
@@ -105,10 +101,6 @@ where
|
||||
|
||||
const C_UNKNOWN: i32 = -1;
|
||||
const C_OFFLINE: i32 = 0;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reachable only from the unused transition client methods below (backlog#1823)"
|
||||
)]
|
||||
const C_ONLINE: i32 = 1;
|
||||
|
||||
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
||||
@@ -328,10 +320,6 @@ impl TransitionClient {
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn endpoint_url(&self) -> Url {
|
||||
self.endpoint_url.clone()
|
||||
}
|
||||
@@ -360,20 +348,12 @@ impl TransitionClient {
|
||||
.to_string())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn trace_errors_only_off(&self) {
|
||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||
*trace_errors_only = false;
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn trace_off(&self) {
|
||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||
*is_trace_enabled = false;
|
||||
@@ -383,20 +363,12 @@ impl TransitionClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||
*endpoint = accelerate_endpoint.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||
*dual_stack = enabled;
|
||||
@@ -426,18 +398,10 @@ impl TransitionClient {
|
||||
(hash_algos, hash_sums)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn is_online(&self) -> bool {
|
||||
!self.is_offline()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn mark_offline(&self) {
|
||||
self.health_status
|
||||
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
||||
@@ -447,18 +411,10 @@ impl TransitionClient {
|
||||
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn health_check(hc_duration: Duration) {
|
||||
let _ = hc_duration;
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
||||
let mut resp_trace: Vec<u8>;
|
||||
|
||||
@@ -1146,7 +1102,6 @@ impl Default for ObjectInfo {
|
||||
}
|
||||
|
||||
impl ObjectInfo {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn remote_version(
|
||||
&self,
|
||||
capabilities: ProviderVersionCapabilities,
|
||||
@@ -1387,12 +1342,12 @@ pub(crate) fn to_object_info_for_provider(
|
||||
};
|
||||
|
||||
// Extract checksums
|
||||
let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32);
|
||||
let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C);
|
||||
let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1);
|
||||
let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256);
|
||||
let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME);
|
||||
let checksum_mode = get_header(AMZ_CHECKSUM_MODE);
|
||||
let checksum_crc32 = get_header("x-amz-checksum-crc32");
|
||||
let checksum_crc32c = get_header("x-amz-checksum-crc32c");
|
||||
let checksum_sha1 = get_header("x-amz-checksum-sha1");
|
||||
let checksum_sha256 = get_header("x-amz-checksum-sha256");
|
||||
let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
|
||||
let checksum_mode = get_header("x-amz-checksum-mode");
|
||||
|
||||
// Build and return the ObjectInfo struct
|
||||
Ok(ObjectInfo {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user