mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 10:43:15 +00:00
Compare commits
68 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65035481f6 | |||
| b2ff43eb1a | |||
| 355c8d2e22 | |||
| 60eb139db9 | |||
| 9ef059c908 | |||
| 51497cb533 | |||
| c7a29ec0a7 | |||
| deb0edb7cc | |||
| a08de9229b | |||
| abffa5cf1b | |||
| b825c54850 | |||
| de9145e87a | |||
| 84bd76a3ce | |||
| 9a2d06b370 | |||
| 00de43528c | |||
| daecb93139 | |||
| 35a30cd614 | |||
| 360bceafce | |||
| 7cb91a0190 | |||
| beb6e1383e | |||
| 59b7d13095 | |||
| e0b87b0e7e | |||
| 21c2fb42bb | |||
| 984c705713 | |||
| 23b17c2d5a | |||
| 89e2513205 | |||
| d091554ffe | |||
| c04ee41cf0 | |||
| a9691b6797 | |||
| 7db3882777 | |||
| 3f3e3f4f05 | |||
| 01e0af6312 | |||
| 3377688dab | |||
| 890ddea94b | |||
| 9f02ca6c36 | |||
| 33cd11472a | |||
| 3ff250f1cd | |||
| d795729585 | |||
| 9e6e02ea09 | |||
| 39274fc37c | |||
| 33eff4c3c4 | |||
| a2f16aa066 | |||
| 4c8b9f87e1 | |||
| 3272730c13 | |||
| 1862112d0c | |||
| cd0ac02879 | |||
| 6cf9cf7bb5 | |||
| f1f86ee9d0 | |||
| 1eef0de003 | |||
| a118d7e4fd | |||
| ed1bedf1fb | |||
| 4392f94e1a | |||
| 81d7b7d07a | |||
| e26668e62c | |||
| 8d3511c1b3 | |||
| d172d05e86 | |||
| 0d86c50760 | |||
| 526d6f667e | |||
| dcf3e4b9e8 | |||
| 04b9c8fd36 | |||
| c1f66969d7 | |||
| cfa9276fad | |||
| db8f55cb97 | |||
| 7f23a1ba91 | |||
| 1619c4be60 | |||
| 72fd7339c9 | |||
| 71e83aeec4 | |||
| 9138c24571 |
@@ -94,6 +94,7 @@ jobs:
|
|||||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||||
|
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
@@ -118,6 +119,7 @@ jobs:
|
|||||||
short_sha=""
|
short_sha=""
|
||||||
is_prerelease=false
|
is_prerelease=false
|
||||||
create_latest=false
|
create_latest=false
|
||||||
|
source_ref="$GITHUB_SHA"
|
||||||
|
|
||||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||||
# Triggered by build workflow completion
|
# Triggered by build workflow completion
|
||||||
@@ -137,6 +139,7 @@ jobs:
|
|||||||
# Extract version info from commit message or use commit SHA
|
# Extract version info from commit message or use commit SHA
|
||||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||||
|
source_ref="$HEAD_SHA"
|
||||||
|
|
||||||
# Determine build type based on triggering workflow event and ref
|
# Determine build type based on triggering workflow event and ref
|
||||||
triggering_event="$TRIGGERING_EVENT"
|
triggering_event="$TRIGGERING_EVENT"
|
||||||
@@ -261,6 +264,23 @@ jobs:
|
|||||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||||
|
tag_ref="refs/tags/$input_version"
|
||||||
|
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||||
|
if [[ "$input_version" == v* ]]; then
|
||||||
|
tag_ref="refs/tags/${input_version#v}"
|
||||||
|
else
|
||||||
|
tag_ref="refs/tags/v$input_version"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||||
|
echo "❌ Release tag not found for Docker build: $input_version"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
source_ref="$tag_ref"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -271,6 +291,7 @@ jobs:
|
|||||||
echo "short_sha=$short_sha"
|
echo "short_sha=$short_sha"
|
||||||
echo "is_prerelease=$is_prerelease"
|
echo "is_prerelease=$is_prerelease"
|
||||||
echo "create_latest=$create_latest"
|
echo "create_latest=$create_latest"
|
||||||
|
echo "source_ref=$source_ref"
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
echo "🐳 Docker Build Summary:"
|
echo "🐳 Docker Build Summary:"
|
||||||
@@ -281,6 +302,7 @@ jobs:
|
|||||||
echo " - Short SHA: $short_sha"
|
echo " - Short SHA: $short_sha"
|
||||||
echo " - Is prerelease: $is_prerelease"
|
echo " - Is prerelease: $is_prerelease"
|
||||||
echo " - Create latest: $create_latest"
|
echo " - Create latest: $create_latest"
|
||||||
|
echo " - Source ref: $source_ref"
|
||||||
|
|
||||||
# Build multi-arch Docker images
|
# Build multi-arch Docker images
|
||||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||||
@@ -308,6 +330,7 @@ jobs:
|
|||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
@@ -397,7 +420,8 @@ jobs:
|
|||||||
LABELS="org.opencontainers.image.title=RustFS"
|
LABELS="org.opencontainers.image.title=RustFS"
|
||||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||||
|
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||||
|
|||||||
@@ -15,28 +15,35 @@
|
|||||||
# Package Workflow - Build DEB/RPM packages
|
# Package Workflow - Build DEB/RPM packages
|
||||||
#
|
#
|
||||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||||
# and uploads them to Cloudflare R2.
|
# and uploads them to Cloudflare R2 and the GitHub release.
|
||||||
#
|
#
|
||||||
# Trigger:
|
# Trigger:
|
||||||
# - release published: automatically package when a GitHub release is published
|
# - workflow_run: automatically package after "Build and Release" completes
|
||||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
# for a release tag (the mac/windows/linux binaries are already uploaded
|
||||||
|
# to the GitHub release before packaging starts)
|
||||||
|
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
||||||
#
|
#
|
||||||
# Flow:
|
# Flow:
|
||||||
# 1. Find the Build workflow run for the release tag
|
# 1. Resolve the triggering Build workflow run for the release tag
|
||||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||||
# 3. Build DEB packages for amd64 and arm64
|
# 3. Build DEB packages for amd64 and arm64
|
||||||
# 4. Build RPM packages for x86_64 and aarch64
|
# 4. Build RPM packages for x86_64 and aarch64
|
||||||
# 5. Upload all packages to Cloudflare R2
|
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
||||||
|
|
||||||
name: Package DEB/RPM
|
name: Package DEB/RPM
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
# contents: write is required to upload packages to the GitHub release
|
||||||
|
contents: write
|
||||||
actions: read
|
actions: read
|
||||||
|
|
||||||
on:
|
on:
|
||||||
release:
|
# Follows the same pattern as docker.yml: run after the release build
|
||||||
types: [ published ]
|
# workflow completes, so packaging is triggered only by release tags
|
||||||
|
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
||||||
|
workflow_run:
|
||||||
|
workflows: [ "Build and Release" ]
|
||||||
|
types: [ completed ]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
tag:
|
tag:
|
||||||
@@ -49,13 +56,26 @@ on:
|
|||||||
type: string
|
type: string
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||||
|
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Resolve which build run to use and extract version info
|
# Resolve which build run to use and extract version info
|
||||||
resolve:
|
resolve:
|
||||||
name: Resolve Build
|
name: Resolve Build
|
||||||
|
# Auto-trigger only from successful tag builds of "Build and Release".
|
||||||
|
# Tag pushes arrive as event == push with head_branch != main (a
|
||||||
|
# non-main push head_branch is the release tag name). Manual dispatch
|
||||||
|
# stays available as a fallback for backfills and re-runs.
|
||||||
|
if: >-
|
||||||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
|
(github.event.workflow_run.conclusion == 'success' &&
|
||||||
|
github.event.workflow_run.event == 'push' &&
|
||||||
|
github.event.workflow_run.head_branch != 'main')
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
outputs:
|
outputs:
|
||||||
@@ -75,8 +95,8 @@ jobs:
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Determine tag
|
# Determine tag
|
||||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||||
TAG="${{ github.event.release.tag_name }}"
|
TAG="${HEAD_BRANCH}"
|
||||||
elif [[ -n "$INPUT_TAG" ]]; then
|
elif [[ -n "$INPUT_TAG" ]]; then
|
||||||
TAG="$INPUT_TAG"
|
TAG="$INPUT_TAG"
|
||||||
else
|
else
|
||||||
@@ -93,6 +113,11 @@ jobs:
|
|||||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||||
|
|
||||||
|
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||||
|
# Use the Build and Release run that triggered this workflow
|
||||||
|
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
||||||
|
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
||||||
|
|
||||||
elif [[ -n "$TAG" ]]; then
|
elif [[ -n "$TAG" ]]; then
|
||||||
# Find the build run that produced this tag
|
# Find the build run that produced this tag
|
||||||
echo "Looking for build run for tag: $TAG"
|
echo "Looking for build run for tag: $TAG"
|
||||||
@@ -456,6 +481,54 @@ jobs:
|
|||||||
echo "✅ Latest packages updated"
|
echo "✅ Latest packages updated"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
- name: Upload packages to GitHub Release
|
||||||
|
if: needs.resolve.outputs.tag != ''
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
TAG="${{ needs.resolve.outputs.tag }}"
|
||||||
|
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||||
|
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||||
|
|
||||||
|
# Upload the packages, then refresh the release checksums so the new
|
||||||
|
# assets are covered, matching the binary release flow.
|
||||||
|
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||||
|
if [[ -n "$f" && -f "$f" ]]; then
|
||||||
|
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
||||||
|
gh release upload "$TAG" "$f" --clobber
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
CHECKSUM_DIR="$(mktemp -d)"
|
||||||
|
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||||
|
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||||
|
|
||||||
|
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||||
|
asset="${spec%%:*}"
|
||||||
|
checksum_cmd="${spec##*:}"
|
||||||
|
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||||
|
|
||||||
|
touch "$checksum_file"
|
||||||
|
|
||||||
|
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||||
|
if [[ -n "$f" && -f "$f" ]]; then
|
||||||
|
base="$(basename "$f")"
|
||||||
|
# Remove any stale entry, then append the fresh digest
|
||||||
|
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||||
|
mv "${checksum_file}.tmp" "$checksum_file"
|
||||||
|
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "📤 Updating ${asset} for release ${TAG}..."
|
||||||
|
gh release upload "$TAG" "$checksum_file" --clobber
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "✅ GitHub release assets updated"
|
||||||
|
|
||||||
# Summary
|
# Summary
|
||||||
summary:
|
summary:
|
||||||
name: Summary
|
name: Summary
|
||||||
|
|||||||
Generated
+115
-103
@@ -278,6 +278,7 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bigdecimal",
|
"bigdecimal",
|
||||||
"bon",
|
"bon",
|
||||||
|
"crc32fast",
|
||||||
"digest 0.11.3",
|
"digest 0.11.3",
|
||||||
"log",
|
"log",
|
||||||
"miniz_oxide 0.9.1",
|
"miniz_oxide 0.9.1",
|
||||||
@@ -289,9 +290,11 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"snap",
|
||||||
"strum",
|
"strum",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
"zstd",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -961,9 +964,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-kms"
|
name = "aws-sdk-kms"
|
||||||
version = "1.114.0"
|
version = "1.115.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
|
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -987,9 +990,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-s3"
|
name = "aws-sdk-s3"
|
||||||
version = "1.141.0"
|
version = "1.142.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
|
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1024,9 +1027,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-sso"
|
name = "aws-sdk-sso"
|
||||||
version = "1.105.0"
|
version = "1.106.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516"
|
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1050,9 +1053,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-ssooidc"
|
name = "aws-sdk-ssooidc"
|
||||||
version = "1.107.0"
|
version = "1.108.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c"
|
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1076,9 +1079,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-sts"
|
name = "aws-sdk-sts"
|
||||||
version = "1.110.0"
|
version = "1.111.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13"
|
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1595,7 +1598,7 @@ version = "0.10.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1614,7 +1617,7 @@ version = "0.3.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1855,9 +1858,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.4.2"
|
version = "1.4.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
|
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
"jobserver",
|
"jobserver",
|
||||||
@@ -1965,7 +1968,7 @@ version = "0.4.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crypto-common 0.1.6",
|
"crypto-common 0.1.7",
|
||||||
"inout 0.1.4",
|
"inout 0.1.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2425,7 +2428,7 @@ version = "0.5.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -2450,11 +2453,11 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.6"
|
version = "0.1.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3661,7 +3664,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer 0.10.4",
|
"block-buffer 0.10.4",
|
||||||
"const-oid 0.9.6",
|
"const-oid 0.9.6",
|
||||||
"crypto-common 0.1.6",
|
"crypto-common 0.1.7",
|
||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3921,7 +3924,7 @@ dependencies = [
|
|||||||
"crypto-bigint 0.5.5",
|
"crypto-bigint 0.5.5",
|
||||||
"digest 0.10.7",
|
"digest 0.10.7",
|
||||||
"ff 0.13.1",
|
"ff 0.13.1",
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
"group 0.13.0",
|
"group 0.13.0",
|
||||||
"hkdf 0.12.4",
|
"hkdf 0.12.4",
|
||||||
"pem-rfc7468 0.7.0",
|
"pem-rfc7468 0.7.0",
|
||||||
@@ -4145,9 +4148,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.10"
|
version = "0.1.11"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de"
|
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "findshlibs"
|
name = "findshlibs"
|
||||||
@@ -4366,9 +4369,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.9"
|
version = "0.14.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"typenum",
|
"typenum",
|
||||||
"version_check",
|
"version_check",
|
||||||
@@ -4377,11 +4380,11 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "1.4.4"
|
version = "1.4.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
"rustversion",
|
"rustversion",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
@@ -4723,9 +4726,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.4.15"
|
version = "0.4.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atomic-waker",
|
"atomic-waker",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -5025,9 +5028,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath"
|
name = "hotpath"
|
||||||
version = "0.23.2"
|
version = "0.23.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2"
|
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-channel",
|
"async-channel",
|
||||||
@@ -5059,9 +5062,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-macros"
|
name = "hotpath-macros"
|
||||||
version = "0.23.2"
|
version = "0.23.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341"
|
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
@@ -5070,15 +5073,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-macros-meta"
|
name = "hotpath-macros-meta"
|
||||||
version = "0.23.2"
|
version = "0.23.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38"
|
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-meta"
|
name = "hotpath-meta"
|
||||||
version = "0.23.2"
|
version = "0.23.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f"
|
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hotpath-macros-meta",
|
"hotpath-macros-meta",
|
||||||
]
|
]
|
||||||
@@ -5277,9 +5280,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_collections"
|
name = "icu_collections"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
|
checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"potential_utf",
|
"potential_utf",
|
||||||
@@ -5291,9 +5294,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_locale_core"
|
name = "icu_locale_core"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
|
checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"litemap",
|
"litemap",
|
||||||
@@ -5304,9 +5307,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_normalizer"
|
name = "icu_normalizer"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
|
checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"icu_collections",
|
"icu_collections",
|
||||||
"icu_normalizer_data",
|
"icu_normalizer_data",
|
||||||
@@ -5318,16 +5321,17 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_normalizer_data"
|
name = "icu_normalizer_data"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
|
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_properties"
|
name = "icu_properties"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
|
checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
"icu_collections",
|
"icu_collections",
|
||||||
"icu_locale_core",
|
"icu_locale_core",
|
||||||
"icu_properties_data",
|
"icu_properties_data",
|
||||||
@@ -5338,15 +5342,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_properties_data"
|
name = "icu_properties_data"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
|
checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_provider"
|
name = "icu_provider"
|
||||||
version = "2.2.0"
|
version = "2.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
|
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"icu_locale_core",
|
"icu_locale_core",
|
||||||
@@ -5414,7 +5418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"block-padding 0.3.3",
|
"block-padding 0.3.3",
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5965,9 +5969,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libredox"
|
name = "libredox"
|
||||||
version = "0.1.19"
|
version = "0.1.20"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
|
checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
@@ -6030,9 +6034,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "litemap"
|
name = "litemap"
|
||||||
version = "0.8.2"
|
version = "0.8.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "local-ip-address"
|
name = "local-ip-address"
|
||||||
@@ -6479,9 +6483,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mqttbytes-core-next"
|
name = "mqttbytes-core-next"
|
||||||
version = "0.33.3"
|
version = "0.34.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c"
|
checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
@@ -6882,7 +6886,7 @@ version = "5.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.21.7",
|
"base64 0.22.1",
|
||||||
"chrono",
|
"chrono",
|
||||||
"getrandom 0.2.17",
|
"getrandom 0.2.17",
|
||||||
"http 1.5.0",
|
"http 1.5.0",
|
||||||
@@ -7341,9 +7345,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pageant"
|
name = "pageant"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5"
|
checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base16ct 1.0.0",
|
"base16ct 1.0.0",
|
||||||
"byteorder",
|
"byteorder",
|
||||||
@@ -7725,9 +7729,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pkg-config"
|
name = "pkg-config"
|
||||||
version = "0.3.33"
|
version = "0.3.34"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "plotters"
|
name = "plotters"
|
||||||
@@ -7833,9 +7837,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "potential_utf"
|
name = "potential_utf"
|
||||||
version = "0.1.5"
|
version = "0.1.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
|
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"zerovec",
|
"zerovec",
|
||||||
]
|
]
|
||||||
@@ -8043,7 +8047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"itertools 0.10.5",
|
"itertools 0.14.0",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
@@ -8063,7 +8067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"itertools 0.10.5",
|
"itertools 0.14.0",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"petgraph 0.8.3",
|
"petgraph 0.8.3",
|
||||||
@@ -8084,7 +8088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.10.5",
|
"itertools 0.14.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
@@ -8097,7 +8101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.10.5",
|
"itertools 0.14.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
@@ -8213,7 +8217,7 @@ dependencies = [
|
|||||||
"reqwest",
|
"reqwest",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"spin 0.12.2",
|
"spin 0.12.3",
|
||||||
"symbolic-demangle",
|
"symbolic-demangle",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
@@ -8282,9 +8286,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quinn-proto"
|
name = "quinn-proto"
|
||||||
version = "0.11.16"
|
version = "0.11.17"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -8550,9 +8554,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redis"
|
name = "redis"
|
||||||
version = "1.5.0"
|
version = "1.6.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"arcstr",
|
"arcstr",
|
||||||
@@ -8564,7 +8568,7 @@ dependencies = [
|
|||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"itoa",
|
"itoa",
|
||||||
"num-bigint 0.4.8",
|
"num-bigint 0.5.1",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"rustls",
|
"rustls",
|
||||||
@@ -8865,9 +8869,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-core-next"
|
name = "rumqttc-core-next"
|
||||||
version = "0.33.3"
|
version = "0.34.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c"
|
checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-tungstenite",
|
"async-tungstenite",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
@@ -8883,18 +8887,18 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-next"
|
name = "rumqttc-next"
|
||||||
version = "0.33.3"
|
version = "0.34.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08"
|
checksum = "477c9bbfba8f3aecc7aad31c6de2eacb75822efaa18e7aeecb8d3d8e534fbf07"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rumqttc-v5-next",
|
"rumqttc-v5-next",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-v5-next"
|
name = "rumqttc-v5-next"
|
||||||
version = "0.33.3"
|
version = "0.34.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc"
|
checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-tungstenite",
|
"async-tungstenite",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -8917,9 +8921,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "russh"
|
name = "russh"
|
||||||
version = "0.62.6"
|
version = "0.62.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5"
|
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.9.2",
|
"aes 0.9.2",
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
@@ -8942,7 +8946,7 @@ dependencies = [
|
|||||||
"enum_dispatch",
|
"enum_dispatch",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures",
|
"futures",
|
||||||
"generic-array 1.4.4",
|
"generic-array 1.4.5",
|
||||||
"getrandom 0.4.3",
|
"getrandom 0.4.3",
|
||||||
"ghash",
|
"ghash",
|
||||||
"hex-literal",
|
"hex-literal",
|
||||||
@@ -9200,6 +9204,7 @@ dependencies = [
|
|||||||
"serial_test",
|
"serial_test",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"shadow-rs",
|
"shadow-rs",
|
||||||
|
"snap",
|
||||||
"socket2",
|
"socket2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
@@ -9276,6 +9281,7 @@ dependencies = [
|
|||||||
"s3s",
|
"s3s",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"smallvec",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tonic",
|
"tonic",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -9486,7 +9492,7 @@ dependencies = [
|
|||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rayon",
|
"rayon",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"spin 0.12.2",
|
"spin 0.12.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -9821,14 +9827,19 @@ name = "rustfs-madmin"
|
|||||||
version = "1.0.0-rc.2"
|
version = "1.0.0-rc.2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hotpath",
|
"hotpath",
|
||||||
|
"http 1.5.0",
|
||||||
"humantime",
|
"humantime",
|
||||||
"hyper",
|
"hyper",
|
||||||
"jiff",
|
"jiff",
|
||||||
|
"reqwest",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
|
"rustfs-signer",
|
||||||
|
"s3s",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
"time",
|
"time",
|
||||||
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -10242,6 +10253,7 @@ dependencies = [
|
|||||||
"rustfs-ecstore",
|
"rustfs-ecstore",
|
||||||
"rustfs-filemeta",
|
"rustfs-filemeta",
|
||||||
"rustfs-lock",
|
"rustfs-lock",
|
||||||
|
"rustfs-s3-types",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
"s3s",
|
"s3s",
|
||||||
@@ -10822,7 +10834,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"base16ct 0.2.0",
|
"base16ct 0.2.0",
|
||||||
"der 0.7.10",
|
"der 0.7.10",
|
||||||
"generic-array 0.14.9",
|
"generic-array 0.14.7",
|
||||||
"pkcs8 0.10.2",
|
"pkcs8 0.10.2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -11386,9 +11398,9 @@ checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "spin"
|
name = "spin"
|
||||||
version = "0.12.2"
|
version = "0.12.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b"
|
checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"lock_api",
|
"lock_api",
|
||||||
]
|
]
|
||||||
@@ -11800,7 +11812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fastrand",
|
"fastrand",
|
||||||
"getrandom 0.3.4",
|
"getrandom 0.4.3",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix",
|
"rustix",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
@@ -11964,9 +11976,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinystr"
|
name = "tinystr"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
|
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"zerovec",
|
"zerovec",
|
||||||
@@ -12640,9 +12652,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "uuid"
|
name = "uuid"
|
||||||
version = "1.24.0"
|
version = "1.24.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"getrandom 0.4.3",
|
"getrandom 0.4.3",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
@@ -13157,9 +13169,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "writeable"
|
name = "writeable"
|
||||||
version = "0.6.3"
|
version = "0.6.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "x509-cert"
|
name = "x509-cert"
|
||||||
@@ -13339,9 +13351,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerotrie"
|
name = "zerotrie"
|
||||||
version = "0.2.4"
|
version = "0.2.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
|
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"yoke",
|
"yoke",
|
||||||
@@ -13350,9 +13362,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerovec"
|
name = "zerovec"
|
||||||
version = "0.11.6"
|
version = "0.11.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
|
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"yoke",
|
"yoke",
|
||||||
"zerofrom",
|
"zerofrom",
|
||||||
@@ -13361,13 +13373,13 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerovec-derive"
|
name = "zerovec-derive"
|
||||||
version = "0.11.3"
|
version = "0.11.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+9
-9
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
|||||||
tower-http = { version = "0.7.0" }
|
tower-http = { version = "0.7.0" }
|
||||||
|
|
||||||
# Serialization and Data Formats
|
# Serialization and Data Formats
|
||||||
apache-avro = "0.22.0"
|
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||||
bytes = { version = "1.12.1" }
|
bytes = { version = "1.12.1" }
|
||||||
bytesize = "2.7.0"
|
bytesize = "2.7.0"
|
||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
@@ -228,9 +228,9 @@ atoi = "3.1.0"
|
|||||||
atomic_enum = "0.3.0"
|
atomic_enum = "0.3.0"
|
||||||
aws-config = { version = "1.10.1" }
|
aws-config = { version = "1.10.1" }
|
||||||
aws-credential-types = { version = "1.3.0" }
|
aws-credential-types = { version = "1.3.0" }
|
||||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||||
aws-smithy-types = { version = "1.6.2" }
|
aws-smithy-types = { version = "1.6.2" }
|
||||||
@@ -284,8 +284,8 @@ rayon = "1.12.0"
|
|||||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||||
reed-solomon-simd = "3.1.0"
|
reed-solomon-simd = "3.1.0"
|
||||||
regex = { version = "1.13.1" }
|
regex = { version = "1.13.1" }
|
||||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||||
redis = { version = "1.5.0" }
|
redis = { version = "1.6.0" }
|
||||||
rustify = { version = "0.7", default-features = false }
|
rustify = { version = "0.7", default-features = false }
|
||||||
rustix = { version = "1.1.4" }
|
rustix = { version = "1.1.4" }
|
||||||
rust-embed = { version = "8.12.0" }
|
rust-embed = { version = "8.12.0" }
|
||||||
@@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" }
|
|||||||
transform-stream = "0.3.1"
|
transform-stream = "0.3.1"
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
urlencoding = "2.1.3"
|
urlencoding = "2.1.3"
|
||||||
uuid = { version = "1.24.0" }
|
uuid = { version = "1.24.1" }
|
||||||
vaultrs = { version = "0.8.0" }
|
vaultrs = { version = "0.8.0" }
|
||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
walkdir = "2.5.0"
|
walkdir = "2.5.0"
|
||||||
@@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" }
|
|||||||
unftp-core = "0.1.0"
|
unftp-core = "0.1.0"
|
||||||
suppaftp = { version = "10.0.1" }
|
suppaftp = { version = "10.0.1" }
|
||||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||||
russh = { version = "0.62.6" }
|
russh = { version = "0.62.7" }
|
||||||
russh-sftp = "2.4.0"
|
russh-sftp = "2.4.0"
|
||||||
|
|
||||||
# WebDAV
|
# WebDAV
|
||||||
@@ -350,7 +350,7 @@ dav-server = "0.11.0"
|
|||||||
# Performance Analysis and Memory Profiling
|
# Performance Analysis and Memory Profiling
|
||||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||||
hotpath = { version = "0.23.2", default-features = false }
|
hotpath = { version = "0.23.3", default-features = false }
|
||||||
# Snapshot testing for output format regression detection
|
# Snapshot testing for output format regression detection
|
||||||
insta = { version = "1.48" }
|
insta = { version = "1.48" }
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ mak = "mak"
|
|||||||
gae = "gae"
|
gae = "gae"
|
||||||
GAE = "GAE"
|
GAE = "GAE"
|
||||||
thr = "thr"
|
thr = "thr"
|
||||||
|
mis = "mis"
|
||||||
# s3-tests original test names (cannot be changed)
|
# s3-tests original test names (cannot be changed)
|
||||||
nonexisted = "nonexisted"
|
nonexisted = "nonexisted"
|
||||||
consts = "consts"
|
consts = "consts"
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
use base64_simd::STANDARD;
|
use base64_simd::STANDARD;
|
||||||
|
|
||||||
|
|||||||
@@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
|||||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||||
|
|
||||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||||
|
|||||||
@@ -476,13 +476,19 @@ impl Checksum for Xxhash64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
struct Md5 {
|
struct Md5 {
|
||||||
hasher: md5::Md5,
|
hasher: md5::Md5,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
impl Md5 {
|
impl Md5 {
|
||||||
fn update(&mut self, bytes: &[u8]) {
|
fn update(&mut self, bytes: &[u8]) {
|
||||||
use md5::Digest;
|
use md5::Digest;
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] }
|
|||||||
jiff = { workspace = true, features = ["serde"] }
|
jiff = { workspace = true, features = ["serde"] }
|
||||||
metrics = { workspace = true }
|
metrics = { workspace = true }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
smallvec = { workspace = true }
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
s3s = { workspace = true, features = ["minio"] }
|
s3s = { workspace = true, features = ["minio"] }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
|||||||
@@ -287,6 +287,9 @@ pub enum HealRequestSource {
|
|||||||
Scanner,
|
Scanner,
|
||||||
AutoHeal,
|
AutoHeal,
|
||||||
ReadRepair,
|
ReadRepair,
|
||||||
|
/// Mission Repair Feed: intents delivered by error paths and replayed
|
||||||
|
/// from the durable MRF journal.
|
||||||
|
Mrf,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HealRequestSource {
|
impl HealRequestSource {
|
||||||
@@ -297,6 +300,7 @@ impl HealRequestSource {
|
|||||||
Self::Scanner => "scanner",
|
Self::Scanner => "scanner",
|
||||||
Self::AutoHeal => "auto_heal",
|
Self::AutoHeal => "auto_heal",
|
||||||
Self::ReadRepair => "read_repair",
|
Self::ReadRepair => "read_repair",
|
||||||
|
Self::Mrf => "mrf",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,8 +17,10 @@ pub mod globals;
|
|||||||
pub mod heal_channel;
|
pub mod heal_channel;
|
||||||
pub mod last_minute;
|
pub mod last_minute;
|
||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
|
pub mod mrf_channel;
|
||||||
mod readiness;
|
mod readiness;
|
||||||
pub mod table_catalog;
|
pub mod table_catalog;
|
||||||
|
pub mod trace_bus;
|
||||||
|
|
||||||
pub use globals::*;
|
pub use globals::*;
|
||||||
pub use readiness::{GlobalReadiness, SystemStage};
|
pub use readiness::{GlobalReadiness, SystemStage};
|
||||||
|
|||||||
@@ -0,0 +1,203 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
//! Mission Repair Feed (MRF) intent channel.
|
||||||
|
//!
|
||||||
|
//! Producers on error paths (read decode failure, scanner metadata
|
||||||
|
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
||||||
|
//! heal crate through a global bounded channel. Delivery is strictly
|
||||||
|
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
||||||
|
//! (counting it) when the channel is full or uninitialized — losing one heal
|
||||||
|
//! hint is always preferred over stalling an IO path. Durable replay of
|
||||||
|
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
||||||
|
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
||||||
|
|
||||||
|
use std::sync::{
|
||||||
|
Arc, OnceLock,
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
};
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
||||||
|
/// dropping (and counting) intents, never by blocking the producer.
|
||||||
|
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
||||||
|
|
||||||
|
/// Why an intent was produced. Drives the heal priority mapping on the
|
||||||
|
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
||||||
|
/// PartialWrite -> Normal).
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum MrfKind {
|
||||||
|
/// Erasure decode failed while serving a read (read path).
|
||||||
|
DecodeFailure,
|
||||||
|
/// Scanner classified object metadata as corrupt.
|
||||||
|
MetadataCorruption,
|
||||||
|
/// A write left the object with fewer committed shards than the set size.
|
||||||
|
PartialWrite,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MrfKind {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
MrfKind::DecodeFailure => "decode-failure",
|
||||||
|
MrfKind::MetadataCorruption => "metadata-corruption",
|
||||||
|
MrfKind::PartialWrite => "partial-write",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
||||||
|
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
||||||
|
/// intent never re-allocates the strings.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct MrfIntent {
|
||||||
|
pub bucket: Arc<str>,
|
||||||
|
pub object: Arc<str>,
|
||||||
|
/// Version the intent targets, as raw UUID bytes.
|
||||||
|
pub version_id: Option<[u8; 16]>,
|
||||||
|
pub kind: MrfKind,
|
||||||
|
pub enqueued_at_ms: u64,
|
||||||
|
/// Times this intent has already been offered to the heal manager.
|
||||||
|
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
||||||
|
pub attempts: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Consumer-side retry ceiling before an intent is given up on.
|
||||||
|
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
||||||
|
|
||||||
|
impl MrfIntent {
|
||||||
|
/// Rough in-memory footprint used by the queue's byte budget.
|
||||||
|
pub fn estimated_bytes(&self) -> usize {
|
||||||
|
// Struct + strings + version bytes; buckets and objects are usually
|
||||||
|
// far below this bound, so rounding up keeps the budget conservative.
|
||||||
|
64 + self.bucket.len() + self.object.len()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
||||||
|
|
||||||
|
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
||||||
|
/// this before touching the channel so the disabled path stays allocation- and
|
||||||
|
/// sync-free.
|
||||||
|
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
||||||
|
|
||||||
|
/// Override delivery (used at heal-runtime startup from configuration).
|
||||||
|
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
||||||
|
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether producers currently deliver intents.
|
||||||
|
pub fn mrf_delivery_enabled() -> bool {
|
||||||
|
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create the global MRF channel and return the consumer half. Fails if the
|
||||||
|
/// channel is already initialized (the heal runtime is a singleton).
|
||||||
|
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
||||||
|
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
||||||
|
GLOBAL_MRF_SENDER
|
||||||
|
.set(sender)
|
||||||
|
.map_err(|_| "MRF channel sender already initialized")?;
|
||||||
|
Ok(receiver)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort, non-blocking intent delivery from an error path.
|
||||||
|
///
|
||||||
|
/// Returns `true` when the intent was accepted into the channel. `false`
|
||||||
|
/// means the intent was dropped (feature disabled, channel not yet
|
||||||
|
/// initialized, or channel full) — callers must not retry or await; the
|
||||||
|
/// existing read-repair / scanner heal paths remain the safety net.
|
||||||
|
///
|
||||||
|
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
||||||
|
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
||||||
|
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
||||||
|
if !mrf_delivery_enabled() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let intent = MrfIntent {
|
||||||
|
bucket: Arc::from(bucket),
|
||||||
|
object: Arc::from(object),
|
||||||
|
version_id: version_id.map(|vid| *vid.as_bytes()),
|
||||||
|
kind,
|
||||||
|
enqueued_at_ms: unix_now_ms(),
|
||||||
|
attempts: 0,
|
||||||
|
};
|
||||||
|
sender.try_send(intent).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_now_ms() -> u64 {
|
||||||
|
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
||||||
|
// failure would be a bug rather than something to handle here.
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_millis() as u64)
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn intents_estimate_is_conservative() {
|
||||||
|
let intent = MrfIntent {
|
||||||
|
bucket: Arc::from("bucket"),
|
||||||
|
object: Arc::from("object"),
|
||||||
|
version_id: Some([0u8; 16]),
|
||||||
|
kind: MrfKind::DecodeFailure,
|
||||||
|
enqueued_at_ms: 0,
|
||||||
|
attempts: 0,
|
||||||
|
};
|
||||||
|
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn try_send_delivers_and_respects_capacity() {
|
||||||
|
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||||
|
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
||||||
|
|
||||||
|
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
||||||
|
let intent = receiver.recv().await.expect("intent should arrive");
|
||||||
|
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
||||||
|
assert_eq!(intent.bucket.as_ref(), "b");
|
||||||
|
|
||||||
|
// Disable delivery: producers become no-ops.
|
||||||
|
set_mrf_delivery_enabled(false);
|
||||||
|
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
||||||
|
set_mrf_delivery_enabled(true);
|
||||||
|
|
||||||
|
// Fill the bounded channel past capacity: excess intents are dropped,
|
||||||
|
// never blocking.
|
||||||
|
let mut accepted = 0;
|
||||||
|
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||||
|
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
||||||
|
accepted += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn try_send_without_channel_is_false() {
|
||||||
|
// This test may run after the tokio test above in the same process;
|
||||||
|
// the singleton semantics make a clean "uninitialized" case hard, so
|
||||||
|
// assert the flag-off behavior only.
|
||||||
|
set_mrf_delivery_enabled(false);
|
||||||
|
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||||
|
set_mrf_delivery_enabled(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use smallvec::SmallVec;
|
||||||
|
use std::{
|
||||||
|
sync::{
|
||||||
|
Arc, OnceLock,
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
},
|
||||||
|
time::{Duration, SystemTime},
|
||||||
|
};
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
||||||
|
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
||||||
|
|
||||||
|
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum TraceKind {
|
||||||
|
Heal,
|
||||||
|
Scanner,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceKind {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Heal => "heal",
|
||||||
|
Self::Scanner => "scanner",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum TraceFunc {
|
||||||
|
HealTask,
|
||||||
|
HealBucket,
|
||||||
|
HealObject,
|
||||||
|
HealCheckAbandonedParts,
|
||||||
|
HealErasureSetPage,
|
||||||
|
ScannerFolder,
|
||||||
|
ScannerIlmAction,
|
||||||
|
ScannerHealCandidate,
|
||||||
|
Dropped,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceFunc {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::HealTask => "heal.Task",
|
||||||
|
Self::HealBucket => "heal.Bucket",
|
||||||
|
Self::HealObject => "heal.Object",
|
||||||
|
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
||||||
|
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
||||||
|
Self::ScannerFolder => "scanner.Folder",
|
||||||
|
Self::ScannerIlmAction => "scanner.IlmAction",
|
||||||
|
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
||||||
|
Self::Dropped => "trace.Dropped",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum TraceVal {
|
||||||
|
Bool(bool),
|
||||||
|
U64(u64),
|
||||||
|
I64(i64),
|
||||||
|
Str(Arc<str>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<bool> for TraceVal {
|
||||||
|
fn from(value: bool) -> Self {
|
||||||
|
Self::Bool(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<u64> for TraceVal {
|
||||||
|
fn from(value: u64) -> Self {
|
||||||
|
Self::U64(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<i64> for TraceVal {
|
||||||
|
fn from(value: i64) -> Self {
|
||||||
|
Self::I64(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&str> for TraceVal {
|
||||||
|
fn from(value: &str) -> Self {
|
||||||
|
Self::Str(Arc::from(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for TraceVal {
|
||||||
|
fn from(value: String) -> Self {
|
||||||
|
Self::Str(Arc::from(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct TraceAttr {
|
||||||
|
pub key: &'static str,
|
||||||
|
pub value: TraceVal,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct TraceEvent {
|
||||||
|
pub kind: TraceKind,
|
||||||
|
pub func: TraceFunc,
|
||||||
|
pub time: SystemTime,
|
||||||
|
pub bucket: Option<Arc<str>>,
|
||||||
|
pub object: Option<Arc<str>>,
|
||||||
|
pub duration: Duration,
|
||||||
|
pub bytes: u64,
|
||||||
|
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceEvent {
|
||||||
|
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
||||||
|
Self {
|
||||||
|
kind,
|
||||||
|
func,
|
||||||
|
time: SystemTime::now(),
|
||||||
|
bucket: None,
|
||||||
|
object: None,
|
||||||
|
duration: Duration::ZERO,
|
||||||
|
bytes: 0,
|
||||||
|
attrs: SmallVec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
||||||
|
self.bucket = Some(bucket.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
||||||
|
self.object = Some(object.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_duration(mut self, duration: Duration) -> Self {
|
||||||
|
self.duration = duration;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
||||||
|
self.bytes = bytes;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
||||||
|
self.attrs.push(TraceAttr {
|
||||||
|
key,
|
||||||
|
value: value.into(),
|
||||||
|
});
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct TraceBus {
|
||||||
|
sender: broadcast::Sender<Arc<TraceEvent>>,
|
||||||
|
subscriber_count: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceBus {
|
||||||
|
pub fn new(capacity: usize) -> Self {
|
||||||
|
let capacity = capacity.max(1);
|
||||||
|
let (sender, _receiver) = broadcast::channel(capacity);
|
||||||
|
Self {
|
||||||
|
sender,
|
||||||
|
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn subscriber_count(&self) -> usize {
|
||||||
|
self.subscriber_count.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn subscribe(&self) -> TraceSubscription {
|
||||||
|
let receiver = self.sender.subscribe();
|
||||||
|
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
||||||
|
TraceSubscription {
|
||||||
|
receiver,
|
||||||
|
subscriber_count: Arc::clone(&self.subscriber_count),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
||||||
|
if self.subscriber_count() == 0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.sender.send(Arc::new(build())).is_ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for TraceBus {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct TraceSubscription {
|
||||||
|
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
||||||
|
subscriber_count: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceSubscription {
|
||||||
|
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
||||||
|
self.receiver.recv().await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
||||||
|
self.receiver.try_recv()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TraceSubscription {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn global_trace_bus() -> &'static TraceBus {
|
||||||
|
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn subscribe_trace_events() -> TraceSubscription {
|
||||||
|
global_trace_bus().subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
||||||
|
global_trace_bus().emit(build)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn trace_subscriber_count() -> usize {
|
||||||
|
global_trace_bus().subscriber_count()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_emit_skips_builder_without_subscribers() {
|
||||||
|
let bus = TraceBus::new(4);
|
||||||
|
let built = AtomicUsize::new(0);
|
||||||
|
|
||||||
|
let sent = bus.emit(|| {
|
||||||
|
built.fetch_add(1, Ordering::Relaxed);
|
||||||
|
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(!sent);
|
||||||
|
assert_eq!(built.load(Ordering::Relaxed), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn trace_subscriber_receives_event() {
|
||||||
|
let bus = TraceBus::new(4);
|
||||||
|
let mut subscription = bus.subscribe();
|
||||||
|
|
||||||
|
assert!(bus.emit(|| {
|
||||||
|
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||||
|
.with_bucket("bucket")
|
||||||
|
.with_object("object")
|
||||||
|
.with_duration(Duration::from_millis(7))
|
||||||
|
.with_bytes(11)
|
||||||
|
.with_attr("dry", true)
|
||||||
|
}));
|
||||||
|
|
||||||
|
let event = subscription
|
||||||
|
.recv()
|
||||||
|
.await
|
||||||
|
.expect("subscriber should receive emitted trace event");
|
||||||
|
|
||||||
|
assert_eq!(event.kind, TraceKind::Heal);
|
||||||
|
assert_eq!(event.func, TraceFunc::HealObject);
|
||||||
|
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
||||||
|
assert_eq!(event.object.as_deref(), Some("object"));
|
||||||
|
assert_eq!(event.duration, Duration::from_millis(7));
|
||||||
|
assert_eq!(event.bytes, 11);
|
||||||
|
assert_eq!(
|
||||||
|
event.attrs.as_slice(),
|
||||||
|
&[TraceAttr {
|
||||||
|
key: "dry",
|
||||||
|
value: TraceVal::Bool(true)
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_subscription_drop_decrements_count() {
|
||||||
|
let bus = TraceBus::new(4);
|
||||||
|
let subscription = bus.subscribe();
|
||||||
|
|
||||||
|
assert_eq!(bus.subscriber_count(), 1);
|
||||||
|
drop(subscription);
|
||||||
|
assert_eq!(bus.subscriber_count(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
||||||
|
let bus = TraceBus::new(2);
|
||||||
|
let mut subscription = bus.subscribe();
|
||||||
|
|
||||||
|
for index in 0_u64..4 {
|
||||||
|
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
||||||
|
}
|
||||||
|
|
||||||
|
let err = subscription
|
||||||
|
.recv()
|
||||||
|
.await
|
||||||
|
.expect_err("receiver should observe lag instead of blocking publishers");
|
||||||
|
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,9 +14,8 @@
|
|||||||
|
|
||||||
//! Shared backpressure policy type.
|
//! Shared backpressure policy type.
|
||||||
//!
|
//!
|
||||||
//! The runtime backpressure implementation (byte-watermark pipes and
|
//! This module only carries the watermark policy; the admission primitive it
|
||||||
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
//! projects into lives in `rustfs-io-core`.
|
||||||
//! carries the watermark policy type that implementation shares.
|
|
||||||
|
|
||||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||||
|
|
||||||
|
|||||||
@@ -177,3 +177,31 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
|||||||
|
|
||||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||||
|
|
||||||
|
/// Environment variable that toggles the MRF (mission repair feed) intent
|
||||||
|
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
||||||
|
/// unconsumed intents are replayed from the durable journal after a restart.
|
||||||
|
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
||||||
|
|
||||||
|
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
||||||
|
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
||||||
|
|
||||||
|
/// Environment variable for the MRF journal byte budget. The journal is
|
||||||
|
/// compacted once its on-disk size crosses this bound.
|
||||||
|
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
||||||
|
|
||||||
|
/// Environment variable for the MRF journal replay batch size (intents per
|
||||||
|
/// replay push round).
|
||||||
|
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
||||||
|
|
||||||
|
/// Default behavior keeps the MRF intent pipeline enabled.
|
||||||
|
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
||||||
|
|
||||||
|
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
||||||
|
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
||||||
|
|
||||||
|
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
||||||
|
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Default MRF replay batch size.
|
||||||
|
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
||||||
|
|||||||
@@ -234,6 +234,31 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
|||||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||||
|
|
||||||
|
/// Enable foreground PutObject request admission.
|
||||||
|
///
|
||||||
|
/// This is an experimental, default-off foreground write backpressure gate for
|
||||||
|
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
||||||
|
/// path and only the existing request counters are updated.
|
||||||
|
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
||||||
|
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
||||||
|
|
||||||
|
/// Maximum foreground PutObject requests admitted concurrently per process.
|
||||||
|
///
|
||||||
|
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
||||||
|
/// A value of `0` disables the gate even when the enable flag is present, so a
|
||||||
|
/// partially configured rollout cannot reject every PUT.
|
||||||
|
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
||||||
|
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
||||||
|
|
||||||
|
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
||||||
|
///
|
||||||
|
/// Once this timeout expires the request fails before body ingest/storage
|
||||||
|
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
||||||
|
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||||
|
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
||||||
|
|
||||||
|
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
||||||
|
|
||||||
/// Environment variable for minimum GetObject timeout in seconds.
|
/// Environment variable for minimum GetObject timeout in seconds.
|
||||||
///
|
///
|
||||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||||
|
|||||||
@@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo {
|
|||||||
pub snapshot_complete: bool,
|
pub snapshot_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Prefix-level usage over a raw entry map — the shared core behind
|
||||||
|
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
||||||
|
/// scanner's writer-side cache has the same map type).
|
||||||
|
///
|
||||||
|
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||||
|
/// names come straight off the child keys — no reverse mapping exists or is
|
||||||
|
/// needed. A compacted prefix carries its aggregate but no children, which
|
||||||
|
/// the `compacted` flag reports so callers can say why the breakdown is
|
||||||
|
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
||||||
|
/// was cut (largest first).
|
||||||
|
pub fn prefix_usage_in_cache(
|
||||||
|
cache: &HashMap<String, DataUsageEntry>,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: &str,
|
||||||
|
max_entries: usize,
|
||||||
|
) -> Option<PrefixUsageQuery> {
|
||||||
|
let prefix = prefix.trim_matches('/');
|
||||||
|
let root = if prefix.is_empty() {
|
||||||
|
bucket.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{bucket}/{prefix}")
|
||||||
|
};
|
||||||
|
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||||
|
|
||||||
|
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||||
|
|
||||||
|
let child_prefix = format!("{root}/");
|
||||||
|
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
||||||
|
.children
|
||||||
|
.iter()
|
||||||
|
.filter_map(|child_key| {
|
||||||
|
let child = cache.get(child_key)?;
|
||||||
|
let child_flat = flatten_entry(cache, child, 1)?;
|
||||||
|
// Child keys are literal `bucket/pre/name` paths; a trailing
|
||||||
|
// slash marks a directory object and is display-only here.
|
||||||
|
let name = child_key
|
||||||
|
.strip_prefix(child_prefix.as_str())
|
||||||
|
.unwrap_or(child_key.as_str())
|
||||||
|
.trim_end_matches('/')
|
||||||
|
.to_string();
|
||||||
|
Some(PrefixUsageEntry {
|
||||||
|
prefix: name,
|
||||||
|
usage: PrefixUsageSummary::from_entry(&child_flat),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
sub_prefixes.sort_by(|left, right| {
|
||||||
|
right
|
||||||
|
.usage
|
||||||
|
.size
|
||||||
|
.cmp(&left.usage.size)
|
||||||
|
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||||
|
});
|
||||||
|
let truncated = sub_prefixes.len() > max_entries;
|
||||||
|
sub_prefixes.truncate(max_entries);
|
||||||
|
|
||||||
|
Some(PrefixUsageQuery {
|
||||||
|
usage,
|
||||||
|
compacted: entry.compacted,
|
||||||
|
truncated,
|
||||||
|
sub_prefixes,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
||||||
|
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
||||||
|
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
||||||
|
|
||||||
|
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
||||||
|
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
||||||
|
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
||||||
|
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
||||||
|
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
||||||
|
if depth > PREFIX_USAGE_MAX_DEPTH {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut flattened = DataUsageEntry::default();
|
||||||
|
if !flattened.checked_merge(root) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
flattened.compacted = root.compacted;
|
||||||
|
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
||||||
|
// child edge pointing back at the root's own key is still terminated by
|
||||||
|
// the visited set on first encounter.
|
||||||
|
let mut visited: HashSet<&str> = HashSet::new();
|
||||||
|
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
||||||
|
while let Some((key, child_depth)) = pending.pop() {
|
||||||
|
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let entry = cache.get(key)?;
|
||||||
|
if !flattened.checked_merge(entry) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
||||||
|
}
|
||||||
|
flattened.children.clear();
|
||||||
|
Some(flattened)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flattened counters of one prefix subtree, as returned by
|
||||||
|
/// [`DataUsageCache::prefix_usage`].
|
||||||
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct PrefixUsageSummary {
|
||||||
|
pub size: u64,
|
||||||
|
pub objects: u64,
|
||||||
|
pub versions: u64,
|
||||||
|
pub delete_markers: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PrefixUsageSummary {
|
||||||
|
fn from_entry(entry: &DataUsageEntry) -> Self {
|
||||||
|
Self {
|
||||||
|
size: entry.size as u64,
|
||||||
|
objects: entry.objects as u64,
|
||||||
|
versions: entry.versions as u64,
|
||||||
|
delete_markers: entry.delete_markers as u64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add another set's counters into this one (entries are partitioned by
|
||||||
|
/// set, so per-set results sum).
|
||||||
|
pub fn merge(&mut self, other: &Self) {
|
||||||
|
self.size = self.size.saturating_add(other.size);
|
||||||
|
self.objects = self.objects.saturating_add(other.objects);
|
||||||
|
self.versions = self.versions.saturating_add(other.versions);
|
||||||
|
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||||
|
pub struct PrefixUsageEntry {
|
||||||
|
pub prefix: String,
|
||||||
|
pub usage: PrefixUsageSummary,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of [`DataUsageCache::prefix_usage`].
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct PrefixUsageQuery {
|
||||||
|
pub usage: PrefixUsageSummary,
|
||||||
|
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
||||||
|
/// but no sub-prefix breakdown exists on disk.
|
||||||
|
pub compacted: bool,
|
||||||
|
/// The breakdown had more entries than `max_entries`; the largest remain.
|
||||||
|
pub truncated: bool,
|
||||||
|
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||||
///
|
///
|
||||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||||
@@ -997,6 +1148,21 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Prefix-level usage for one bucket subtree, plus the one-level
|
||||||
|
/// breakdown below it (rustfs/backlog#1872, MinIO
|
||||||
|
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
||||||
|
/// full counters instead of first-level sizes only).
|
||||||
|
///
|
||||||
|
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||||
|
/// names come straight off the child keys — no reverse mapping exists or
|
||||||
|
/// is needed. A compacted prefix carries its aggregate but no children,
|
||||||
|
/// which the `compacted` flag reports so callers can say why the
|
||||||
|
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
||||||
|
/// `max_entries` and was cut (largest first).
|
||||||
|
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||||
|
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn force_compact(&mut self, limit: usize) {
|
pub fn force_compact(&mut self, limit: usize) {
|
||||||
if self.cache.len() < limit {
|
if self.cache.len() < limit {
|
||||||
return;
|
return;
|
||||||
@@ -1898,6 +2064,126 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
||||||
|
/// distinct counters so aggregation is observable.
|
||||||
|
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
||||||
|
let mut cache = DataUsageCache::default();
|
||||||
|
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
||||||
|
cache.replace(
|
||||||
|
path,
|
||||||
|
parent,
|
||||||
|
DataUsageEntry {
|
||||||
|
size,
|
||||||
|
objects,
|
||||||
|
versions,
|
||||||
|
delete_markers,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
};
|
||||||
|
insert("bucket", "", 0, 0, 0, 0);
|
||||||
|
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
||||||
|
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
||||||
|
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
||||||
|
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
||||||
|
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
||||||
|
let cache = prefix_usage_fixture_cache();
|
||||||
|
|
||||||
|
let root = cache
|
||||||
|
.prefix_usage("bucket", "", 100)
|
||||||
|
.expect("root query must find the bucket entry");
|
||||||
|
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
||||||
|
assert_eq!(root.usage.objects, 5);
|
||||||
|
assert_eq!(root.usage.versions, 7);
|
||||||
|
assert_eq!(root.usage.delete_markers, 2);
|
||||||
|
assert!(!root.compacted);
|
||||||
|
assert!(!root.truncated);
|
||||||
|
// Breakdown is one level: b (240) before a (100) before loose (10),
|
||||||
|
// each flattened to its own subtree total.
|
||||||
|
let names: Vec<(&str, u64)> = root
|
||||||
|
.sub_prefixes
|
||||||
|
.iter()
|
||||||
|
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
||||||
|
let cache = prefix_usage_fixture_cache();
|
||||||
|
|
||||||
|
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
||||||
|
assert_eq!(b.usage.size, 240);
|
||||||
|
assert_eq!(b.usage.versions, 5);
|
||||||
|
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||||
|
assert_eq!(names, vec!["c", "d"]);
|
||||||
|
|
||||||
|
// Prefix slashes are normalized away.
|
||||||
|
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
||||||
|
assert_eq!(slashed.usage.size, 240);
|
||||||
|
|
||||||
|
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
||||||
|
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prefix_usage_reports_and_respects_truncation() {
|
||||||
|
let cache = prefix_usage_fixture_cache();
|
||||||
|
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
||||||
|
assert!(capped.truncated, "three children capped to two must flag truncation");
|
||||||
|
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||||
|
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prefix_usage_marks_compacted_entries() {
|
||||||
|
let mut cache = DataUsageCache::default();
|
||||||
|
cache.replace(
|
||||||
|
"bucket",
|
||||||
|
"",
|
||||||
|
DataUsageEntry {
|
||||||
|
size: 999,
|
||||||
|
objects: 9,
|
||||||
|
compacted: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
||||||
|
assert!(compacted.compacted, "compaction must be visible to callers");
|
||||||
|
assert_eq!(compacted.usage.size, 999);
|
||||||
|
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
||||||
|
// A self-referencing child (corrupt cache) must yield a miss for the
|
||||||
|
// whole query, not unbounded recursion.
|
||||||
|
let mut cache = prefix_usage_fixture_cache();
|
||||||
|
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
||||||
|
entry.children.insert("bucket/b".to_string());
|
||||||
|
}
|
||||||
|
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
||||||
|
// The unaffected sibling still answers.
|
||||||
|
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
||||||
|
|
||||||
|
// A child key with no entry (dangling link) is rejected rather than
|
||||||
|
// silently dropped: half a tree would under-report usage.
|
||||||
|
let mut dangling = prefix_usage_fixture_cache();
|
||||||
|
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
||||||
|
entry.children.insert("bucket/b/ghost".to_string());
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
||||||
|
"a dangling child link must be rejected"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hash_path_uses_portable_slash_semantics() {
|
fn hash_path_uses_portable_slash_semantics() {
|
||||||
for (input, expected) in [
|
for (input, expected) in [
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ use rustfs_signer::sign_v4;
|
|||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::fs as stdfs;
|
use std::fs as stdfs;
|
||||||
|
use std::io::ErrorKind;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
use std::sync::Once;
|
use std::sync::Once;
|
||||||
@@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
|||||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||||
|
const TEST_PORT_MIN: u16 = 20_000;
|
||||||
|
const TEST_PORT_RANGE: u16 = 40_000;
|
||||||
|
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||||
|
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||||
|
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||||
@@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
|||||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct PortAllocatorGuard;
|
||||||
|
|
||||||
|
impl PortAllocatorGuard {
|
||||||
|
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
loop {
|
||||||
|
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
||||||
|
Ok(()) => return Ok(Self),
|
||||||
|
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||||
|
remove_stale_port_allocator_lock();
|
||||||
|
sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
Err(err) => return Err(err.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for PortAllocatorGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn advance_test_port(port: u16) -> u16 {
|
||||||
|
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||||
|
TEST_PORT_MIN + offset
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seeded_test_port() -> u16 {
|
||||||
|
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||||
|
TEST_PORT_MIN + offset
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_next_test_port() -> u16 {
|
||||||
|
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||||
|
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||||
|
.unwrap_or_else(seeded_test_port)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_stale_port_allocator_lock() {
|
||||||
|
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Ok(modified) = metadata.modified() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
||||||
|
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn capture_command_logs(
|
pub(crate) fn capture_command_logs(
|
||||||
command: &mut Command,
|
command: &mut Command,
|
||||||
log_path: Option<&str>,
|
log_path: Option<&str>,
|
||||||
@@ -508,10 +572,21 @@ impl RustFSTestEnvironment {
|
|||||||
/// Find an available port for the test
|
/// Find an available port for the test
|
||||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
let _guard = PortAllocatorGuard::acquire().await?;
|
||||||
let port = listener.local_addr()?.port();
|
let mut next_port = read_next_test_port();
|
||||||
drop(listener);
|
|
||||||
Ok(port)
|
for _ in 0..TEST_PORT_RANGE {
|
||||||
|
let port = next_port;
|
||||||
|
next_port = advance_test_port(next_port);
|
||||||
|
write_next_test_port(next_port)?;
|
||||||
|
|
||||||
|
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||||
|
drop(listener);
|
||||||
|
return Ok(port);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err("no available E2E test port found".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kill any existing RustFS processes
|
/// Kill any existing RustFS processes
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end
|
|||||||
|
|
||||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||||
|
|
||||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||||
|
|
||||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions.
|
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||||
|
|
||||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||||
|
|||||||
@@ -30,10 +30,12 @@ use s3s::access::{S3Access, S3AccessContext};
|
|||||||
use s3s::auth::SimpleAuth;
|
use s3s::auth::SimpleAuth;
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput,
|
||||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput,
|
||||||
|
GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput,
|
||||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||||
|
UploadPartInput, UploadPartOutput,
|
||||||
};
|
};
|
||||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||||
@@ -76,6 +78,25 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
|||||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||||
|
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-tagging-timestamp",
|
||||||
|
"x-minio-source-replication-tagging-timestamp",
|
||||||
|
];
|
||||||
|
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-retention-timestamp",
|
||||||
|
"x-minio-source-replication-retention-timestamp",
|
||||||
|
];
|
||||||
|
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-legalhold-timestamp",
|
||||||
|
"x-minio-source-replication-legalhold-timestamp",
|
||||||
|
];
|
||||||
|
/// Wire prefix of the SSE-C passthrough replication transport headers
|
||||||
|
/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a
|
||||||
|
/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with
|
||||||
|
/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO /
|
||||||
|
/// generic S3, which silently discard unknown x-* headers.
|
||||||
|
const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-";
|
||||||
|
const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm";
|
||||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||||
|
|
||||||
@@ -91,6 +112,9 @@ pub enum Operation {
|
|||||||
GetObject,
|
GetObject,
|
||||||
HeadObject,
|
HeadObject,
|
||||||
DeleteObject,
|
DeleteObject,
|
||||||
|
GetObjectTagging,
|
||||||
|
PutObjectTagging,
|
||||||
|
DeleteObjectTagging,
|
||||||
ListObjectVersions,
|
ListObjectVersions,
|
||||||
CreateMultipartUpload,
|
CreateMultipartUpload,
|
||||||
UploadPart,
|
UploadPart,
|
||||||
@@ -118,6 +142,61 @@ pub enum FaultAction {
|
|||||||
WrongEtag,
|
WrongEtag,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||||
|
/// sender-side tests can assert what a real target would receive.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ReplicationTimestampHeaders {
|
||||||
|
pub tagging: Option<String>,
|
||||||
|
pub retention: Option<String>,
|
||||||
|
pub legalhold: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReplicationTimestampHeaders {
|
||||||
|
fn from_headers(headers: &HeaderMap) -> Self {
|
||||||
|
Self {
|
||||||
|
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read-proxy related headers observed on a request, journaled so proxy
|
||||||
|
/// tests can assert the exact wire contract: the anti-loop marker present,
|
||||||
|
/// the replication-check exemption absent, and the client SSE-C key family
|
||||||
|
/// forwarded verbatim. The SSE-C key value itself is never retained — only
|
||||||
|
/// its presence.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ProxyHeaderSnapshot {
|
||||||
|
pub source_proxy_request: Option<String>,
|
||||||
|
pub replication_check: Option<String>,
|
||||||
|
pub ssec_algorithm: Option<String>,
|
||||||
|
pub ssec_key_present: bool,
|
||||||
|
pub ssec_key_md5: Option<String>,
|
||||||
|
/// Whether the request carried any `X-Rustfs-Replication-*` SSE-C
|
||||||
|
/// passthrough transport header, so fail-closed tests can assert the
|
||||||
|
/// sender really shipped the material a dropping target discarded.
|
||||||
|
pub ssec_transport_present: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProxyHeaderSnapshot {
|
||||||
|
fn from_headers(headers: &HeaderMap) -> Self {
|
||||||
|
Self {
|
||||||
|
source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"])
|
||||||
|
.map(bounded_journal_value),
|
||||||
|
replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"])
|
||||||
|
.map(bounded_journal_value),
|
||||||
|
ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"])
|
||||||
|
.map(bounded_journal_value),
|
||||||
|
ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"),
|
||||||
|
ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value),
|
||||||
|
ssec_transport_present: headers
|
||||||
|
.keys()
|
||||||
|
.any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Credential-free request metadata retained for deterministic assertions.
|
/// Credential-free request metadata retained for deterministic assertions.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct RequestRecord {
|
pub struct RequestRecord {
|
||||||
@@ -131,6 +210,8 @@ pub struct RequestRecord {
|
|||||||
pub part_number: Option<i32>,
|
pub part_number: Option<i32>,
|
||||||
pub content_length: Option<u64>,
|
pub content_length: Option<u64>,
|
||||||
pub consumed_bytes: Option<usize>,
|
pub consumed_bytes: Option<usize>,
|
||||||
|
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||||
|
pub proxy_headers: ProxyHeaderSnapshot,
|
||||||
pub fault: Option<FaultAction>,
|
pub fault: Option<FaultAction>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,6 +227,10 @@ struct ControlState {
|
|||||||
struct StoreState {
|
struct StoreState {
|
||||||
assign_own_version_ids: bool,
|
assign_own_version_ids: bool,
|
||||||
assign_own_multipart_version_ids: bool,
|
assign_own_multipart_version_ids: bool,
|
||||||
|
/// MinIO-like mode: silently discard non-whitelisted replication
|
||||||
|
/// transport headers instead of storing them (see
|
||||||
|
/// [`REPLICATION_SSE_TRANSPORT_PREFIX`]).
|
||||||
|
drop_unlisted_replication_headers: bool,
|
||||||
buckets: HashMap<String, BucketState>,
|
buckets: HashMap<String, BucketState>,
|
||||||
uploads: HashMap<String, MultipartState>,
|
uploads: HashMap<String, MultipartState>,
|
||||||
total_bytes: usize,
|
total_bytes: usize,
|
||||||
@@ -167,6 +252,12 @@ struct ObjectVersion {
|
|||||||
delete_marker: bool,
|
delete_marker: bool,
|
||||||
content_type: Option<String>,
|
content_type: Option<String>,
|
||||||
metadata: Option<HashMap<String, String>>,
|
metadata: Option<HashMap<String, String>>,
|
||||||
|
/// Object tags as ordered key/value pairs (PutObjectTagging replaces the
|
||||||
|
/// whole set, DeleteObjectTagging clears it).
|
||||||
|
tags: Vec<(String, String)>,
|
||||||
|
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||||
|
/// target behavior); empty when the drop mode discarded them.
|
||||||
|
replication_sse_headers: Vec<(String, String)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -176,6 +267,7 @@ struct MultipartState {
|
|||||||
version_id: String,
|
version_id: String,
|
||||||
content_type: Option<String>,
|
content_type: Option<String>,
|
||||||
metadata: Option<HashMap<String, String>>,
|
metadata: Option<HashMap<String, String>>,
|
||||||
|
replication_sse_headers: Vec<(String, String)>,
|
||||||
parts: BTreeMap<i32, MultipartPart>,
|
parts: BTreeMap<i32, MultipartPart>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -396,6 +488,15 @@ impl FakeS3Target {
|
|||||||
|
|
||||||
/// Mint own version ids for the multipart path only — models a target
|
/// Mint own version ids for the multipart path only — models a target
|
||||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||||
|
/// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C
|
||||||
|
/// passthrough transport header instead of storing it. The default (off)
|
||||||
|
/// models a RustFS target, which preserves the headers and echoes SSE-C
|
||||||
|
/// evidence (`x-amz-server-side-encryption-customer-algorithm`) on
|
||||||
|
/// HEAD/GET of the replica.
|
||||||
|
pub fn drop_unlisted_replication_headers(&self, enabled: bool) {
|
||||||
|
lock(&self.backend.store).drop_unlisted_replication_headers = enabled;
|
||||||
|
}
|
||||||
|
|
||||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||||
}
|
}
|
||||||
@@ -536,7 +637,17 @@ impl S3Access for FaultAccess {
|
|||||||
.get(CONTENT_LENGTH)
|
.get(CONTENT_LENGTH)
|
||||||
.and_then(|value| value.to_str().ok())
|
.and_then(|value| value.to_str().ok())
|
||||||
.and_then(|value| value.parse().ok());
|
.and_then(|value| value.parse().ok());
|
||||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||||
|
let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers());
|
||||||
|
let fault = record_request(
|
||||||
|
&self.control,
|
||||||
|
operation,
|
||||||
|
context.method().clone(),
|
||||||
|
parsed,
|
||||||
|
content_length,
|
||||||
|
replication_timestamps,
|
||||||
|
proxy_headers,
|
||||||
|
);
|
||||||
if let Some(RequestFault {
|
if let Some(RequestFault {
|
||||||
action: FaultAction::Status(status),
|
action: FaultAction::Status(status),
|
||||||
..
|
..
|
||||||
@@ -575,6 +686,9 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
|||||||
"GetObject" => Operation::GetObject,
|
"GetObject" => Operation::GetObject,
|
||||||
"HeadObject" => Operation::HeadObject,
|
"HeadObject" => Operation::HeadObject,
|
||||||
"DeleteObject" => Operation::DeleteObject,
|
"DeleteObject" => Operation::DeleteObject,
|
||||||
|
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||||
|
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||||
|
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||||
"UploadPart" => Operation::UploadPart,
|
"UploadPart" => Operation::UploadPart,
|
||||||
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
||||||
@@ -589,6 +703,8 @@ fn record_request(
|
|||||||
method: Method,
|
method: Method,
|
||||||
parsed: ParsedRequest,
|
parsed: ParsedRequest,
|
||||||
content_length: Option<u64>,
|
content_length: Option<u64>,
|
||||||
|
replication_timestamps: ReplicationTimestampHeaders,
|
||||||
|
proxy_headers: ProxyHeaderSnapshot,
|
||||||
) -> Option<RequestFault> {
|
) -> Option<RequestFault> {
|
||||||
let mut state = lock(control);
|
let mut state = lock(control);
|
||||||
let action = parsed
|
let action = parsed
|
||||||
@@ -613,6 +729,8 @@ fn record_request(
|
|||||||
part_number: parsed.part_number,
|
part_number: parsed.part_number,
|
||||||
content_length,
|
content_length,
|
||||||
consumed_bytes: None,
|
consumed_bytes: None,
|
||||||
|
replication_timestamps,
|
||||||
|
proxy_headers,
|
||||||
fault: action.clone(),
|
fault: action.clone(),
|
||||||
});
|
});
|
||||||
action.map(|action| RequestFault { sequence, action })
|
action.map(|action| RequestFault { sequence, action })
|
||||||
@@ -679,6 +797,15 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
|||||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||||
|
(&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||||
|
Operation::GetObjectTagging
|
||||||
|
}
|
||||||
|
(&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||||
|
Operation::PutObjectTagging
|
||||||
|
}
|
||||||
|
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||||
|
Operation::DeleteObjectTagging
|
||||||
|
}
|
||||||
// A replication PUT addresses the source version via `?versionId=`.
|
// A replication PUT addresses the source version via `?versionId=`.
|
||||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||||
@@ -746,6 +873,29 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
|||||||
Ok(version_id.to_string())
|
Ok(version_id.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Capture the SSE-C passthrough transport headers a replication PUT carried.
|
||||||
|
/// Returns an empty set in the MinIO-like drop mode.
|
||||||
|
fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> {
|
||||||
|
if drop_unlisted {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
headers
|
||||||
|
.iter()
|
||||||
|
.filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX))
|
||||||
|
.filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string())))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// SSE-C evidence a RustFS-like target echoes for a stored passthrough
|
||||||
|
/// replica: the customer algorithm restored from the transport headers.
|
||||||
|
fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option<String> {
|
||||||
|
version
|
||||||
|
.replication_sse_headers
|
||||||
|
.iter()
|
||||||
|
.find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER)
|
||||||
|
.map(|(_, value)| value.clone())
|
||||||
|
}
|
||||||
|
|
||||||
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
||||||
header_value(headers, &SOURCE_ETAG_HEADERS)
|
header_value(headers, &SOURCE_ETAG_HEADERS)
|
||||||
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
||||||
@@ -1093,6 +1243,33 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option<
|
|||||||
Ok(version.clone())
|
Ok(version.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Replace (or clear, with an empty vec) the tag set of the addressed
|
||||||
|
/// version, returning its version id. Mirrors `find_version` addressing:
|
||||||
|
/// explicit version id or the latest version, delete markers rejected.
|
||||||
|
fn set_version_tags(
|
||||||
|
state: &mut StoreState,
|
||||||
|
bucket: &str,
|
||||||
|
key: &str,
|
||||||
|
version_id: Option<&str>,
|
||||||
|
tags: Vec<(String, String)>,
|
||||||
|
) -> S3Result<String> {
|
||||||
|
// Resolve first (immutable) so the error paths match find_version.
|
||||||
|
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||||
|
let versions = state
|
||||||
|
.buckets
|
||||||
|
.get_mut(bucket)
|
||||||
|
.expect("bucket existence checked by find_version")
|
||||||
|
.objects
|
||||||
|
.get_mut(key)
|
||||||
|
.expect("key existence checked by find_version");
|
||||||
|
let version = versions
|
||||||
|
.iter_mut()
|
||||||
|
.find(|version| version.version_id == resolved)
|
||||||
|
.expect("version existence checked by find_version");
|
||||||
|
version.tags = tags;
|
||||||
|
Ok(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl S3 for FakeBackend {
|
impl S3 for FakeBackend {
|
||||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||||
@@ -1189,7 +1366,10 @@ impl S3 for FakeBackend {
|
|||||||
let input = req.input;
|
let input = req.input;
|
||||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
let (assign_own, drop_unlisted) = {
|
||||||
|
let state = lock(&self.store);
|
||||||
|
(state.assign_own_version_ids, state.drop_unlisted_replication_headers)
|
||||||
|
};
|
||||||
let version_id = new_version_id(&headers, assign_own)?;
|
let version_id = new_version_id(&headers, assign_own)?;
|
||||||
let e_tag = match source_etag(&headers)? {
|
let e_tag = match source_etag(&headers)? {
|
||||||
Some(value) => value,
|
Some(value) => value,
|
||||||
@@ -1206,6 +1386,8 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
content_type: input.content_type,
|
content_type: input.content_type,
|
||||||
metadata: input.metadata,
|
metadata: input.metadata,
|
||||||
|
tags: Vec::new(),
|
||||||
|
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||||
};
|
};
|
||||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
@@ -1226,6 +1408,7 @@ impl S3 for FakeBackend {
|
|||||||
let state = lock(&self.store);
|
let state = lock(&self.store);
|
||||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||||
};
|
};
|
||||||
|
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
S3Response::new(GetObjectOutput {
|
S3Response::new(GetObjectOutput {
|
||||||
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
||||||
@@ -1235,6 +1418,7 @@ impl S3 for FakeBackend {
|
|||||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||||
last_modified: Some(version.last_modified.clone()),
|
last_modified: Some(version.last_modified.clone()),
|
||||||
version_id: Some(version.version_id),
|
version_id: Some(version.version_id),
|
||||||
|
sse_customer_algorithm,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
fault.as_ref(),
|
fault.as_ref(),
|
||||||
@@ -1249,6 +1433,7 @@ impl S3 for FakeBackend {
|
|||||||
let state = lock(&self.store);
|
let state = lock(&self.store);
|
||||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||||
};
|
};
|
||||||
|
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
S3Response::new(HeadObjectOutput {
|
S3Response::new(HeadObjectOutput {
|
||||||
content_length: Some(version.body.len() as i64),
|
content_length: Some(version.body.len() as i64),
|
||||||
@@ -1257,12 +1442,79 @@ impl S3 for FakeBackend {
|
|||||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||||
last_modified: Some(version.last_modified.clone()),
|
last_modified: Some(version.last_modified.clone()),
|
||||||
version_id: Some(version.version_id),
|
version_id: Some(version.version_id),
|
||||||
|
sse_customer_algorithm,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
fault.as_ref(),
|
fault.as_ref(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||||
|
let fault = request_fault(&req);
|
||||||
|
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||||
|
let input = req.input;
|
||||||
|
let version = {
|
||||||
|
let state = lock(&self.store);
|
||||||
|
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||||
|
};
|
||||||
|
let tag_set: TagSet = version
|
||||||
|
.tags
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, value)| Tag {
|
||||||
|
key: Some(key),
|
||||||
|
value: Some(value),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Ok(apply_response_fault(
|
||||||
|
S3Response::new(GetObjectTaggingOutput {
|
||||||
|
tag_set,
|
||||||
|
version_id: Some(ObjectVersionId::from(version.version_id)),
|
||||||
|
}),
|
||||||
|
fault.as_ref(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||||
|
let fault = request_fault(&req);
|
||||||
|
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||||
|
let input = req.input;
|
||||||
|
let tags = input
|
||||||
|
.tagging
|
||||||
|
.tag_set
|
||||||
|
.into_iter()
|
||||||
|
.map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default()))
|
||||||
|
.collect();
|
||||||
|
let version_id = {
|
||||||
|
let mut state = lock(&self.store);
|
||||||
|
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)?
|
||||||
|
};
|
||||||
|
Ok(apply_response_fault(
|
||||||
|
S3Response::new(PutObjectTaggingOutput {
|
||||||
|
version_id: Some(ObjectVersionId::from(version_id)),
|
||||||
|
}),
|
||||||
|
fault.as_ref(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_object_tagging(
|
||||||
|
&self,
|
||||||
|
req: S3Request<DeleteObjectTaggingInput>,
|
||||||
|
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||||
|
let fault = request_fault(&req);
|
||||||
|
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||||
|
let input = req.input;
|
||||||
|
let version_id = {
|
||||||
|
let mut state = lock(&self.store);
|
||||||
|
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())?
|
||||||
|
};
|
||||||
|
Ok(apply_response_fault(
|
||||||
|
S3Response::new(DeleteObjectTaggingOutput {
|
||||||
|
version_id: Some(ObjectVersionId::from(version_id)),
|
||||||
|
}),
|
||||||
|
fault.as_ref(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||||
let fault = request_fault(&req);
|
let fault = request_fault(&req);
|
||||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||||
@@ -1339,6 +1591,8 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
content_type: None,
|
content_type: None,
|
||||||
metadata: None,
|
metadata: None,
|
||||||
|
tags: Vec::new(),
|
||||||
|
replication_sse_headers: Vec::new(),
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
@@ -1366,9 +1620,10 @@ impl S3 for FakeBackend {
|
|||||||
ensure_upload_budget(&state)?;
|
ensure_upload_budget(&state)?;
|
||||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||||
let upload_id = Uuid::new_v4().to_string();
|
let upload_id = Uuid::new_v4().to_string();
|
||||||
// Read the flag before the mutable borrow of `state.uploads` below
|
// Read the flags before the mutable borrow of `state.uploads` below
|
||||||
// (and never re-lock the store: the mutex is not reentrant).
|
// (and never re-lock the store: the mutex is not reentrant).
|
||||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||||
|
let drop_unlisted = state.drop_unlisted_replication_headers;
|
||||||
let version_id = new_version_id(&headers, mint_own)?;
|
let version_id = new_version_id(&headers, mint_own)?;
|
||||||
state.uploads.insert(
|
state.uploads.insert(
|
||||||
upload_id.clone(),
|
upload_id.clone(),
|
||||||
@@ -1378,6 +1633,7 @@ impl S3 for FakeBackend {
|
|||||||
version_id,
|
version_id,
|
||||||
content_type: input.content_type,
|
content_type: input.content_type,
|
||||||
metadata: input.metadata,
|
metadata: input.metadata,
|
||||||
|
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -1515,6 +1771,7 @@ impl S3 for FakeBackend {
|
|||||||
version_id: upload.version_id.clone(),
|
version_id: upload.version_id.clone(),
|
||||||
content_type: upload.content_type.clone(),
|
content_type: upload.content_type.clone(),
|
||||||
metadata: upload.metadata.clone(),
|
metadata: upload.metadata.clone(),
|
||||||
|
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
selected,
|
selected,
|
||||||
@@ -1541,6 +1798,8 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
content_type: upload.content_type,
|
content_type: upload.content_type,
|
||||||
metadata: upload.metadata,
|
metadata: upload.metadata,
|
||||||
|
tags: Vec::new(),
|
||||||
|
replication_sse_headers: upload.replication_sse_headers,
|
||||||
};
|
};
|
||||||
let mut state = lock(&self.store);
|
let mut state = lock(&self.store);
|
||||||
let current = state
|
let current = state
|
||||||
@@ -1699,6 +1958,111 @@ mod tests {
|
|||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
target.create_bucket("target-bucket");
|
||||||
|
let client = client(&target);
|
||||||
|
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket("target-bucket")
|
||||||
|
.key("plain")
|
||||||
|
.body(ByteStream::from_static(b"plain"))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket("target-bucket")
|
||||||
|
.key("stamped")
|
||||||
|
.body(ByteStream::from_static(b"stamped"))
|
||||||
|
.customize()
|
||||||
|
.map_request(move |mut request| {
|
||||||
|
let headers = request.headers_mut();
|
||||||
|
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||||
|
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||||
|
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||||
|
Ok::<_, std::convert::Infallible>(request)
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let requests = target.requests();
|
||||||
|
let plain = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||||
|
.expect("plain PUT must be journaled");
|
||||||
|
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||||
|
|
||||||
|
let stamped = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||||
|
.expect("stamped PUT must be journaled");
|
||||||
|
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||||
|
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||||
|
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Default mode is RustFS-like: SSE-C passthrough transport headers are
|
||||||
|
/// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is
|
||||||
|
/// MinIO-like: the headers are silently discarded, so no evidence comes
|
||||||
|
/// back — the exact difference the N2 fail-closed audit keys on. Both
|
||||||
|
/// modes journal that the sender shipped the transport headers.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> {
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
target.create_bucket("target-bucket");
|
||||||
|
let client = client(&target);
|
||||||
|
|
||||||
|
let put_with_transport_headers = |key: &'static str| {
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket("target-bucket")
|
||||||
|
.key(key)
|
||||||
|
.body(ByteStream::from_static(b"ciphertext"))
|
||||||
|
.customize()
|
||||||
|
.map_request(move |mut request| {
|
||||||
|
let headers = request.headers_mut();
|
||||||
|
headers.insert("x-rustfs-replication-ssec-algorithm", "AES256");
|
||||||
|
headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA==");
|
||||||
|
Ok::<_, std::convert::Infallible>(request)
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
};
|
||||||
|
|
||||||
|
put_with_transport_headers("kept").await?;
|
||||||
|
let head = client.head_object().bucket("target-bucket").key("kept").send().await?;
|
||||||
|
assert_eq!(head.sse_customer_algorithm(), Some("AES256"));
|
||||||
|
let get = client.get_object().bucket("target-bucket").key("kept").send().await?;
|
||||||
|
assert_eq!(get.sse_customer_algorithm(), Some("AES256"));
|
||||||
|
|
||||||
|
target.drop_unlisted_replication_headers(true);
|
||||||
|
put_with_transport_headers("dropped").await?;
|
||||||
|
let head = client.head_object().bucket("target-bucket").key("dropped").send().await?;
|
||||||
|
assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence");
|
||||||
|
|
||||||
|
let requests = target.requests();
|
||||||
|
for key in ["kept", "dropped"] {
|
||||||
|
let record = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key))
|
||||||
|
.expect("PUT must be journaled");
|
||||||
|
assert!(
|
||||||
|
record.proxy_headers.ssec_transport_present,
|
||||||
|
"the journal must prove the sender shipped the transport headers for {key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let plain_head = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::HeadObject)
|
||||||
|
.expect("HEAD must be journaled");
|
||||||
|
assert!(!plain_head.proxy_headers.ssec_transport_present);
|
||||||
|
|
||||||
|
target.shutdown().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
macro_rules! assert_sdk_error {
|
macro_rules! assert_sdk_error {
|
||||||
($error:expr, $status:expr, $code:expr) => {{
|
($error:expr, $status:expr, $code:expr) => {{
|
||||||
let error = &$error;
|
let error = &$error;
|
||||||
@@ -2964,6 +3328,7 @@ mod tests {
|
|||||||
version_id: index.to_string(),
|
version_id: index.to_string(),
|
||||||
content_type: None,
|
content_type: None,
|
||||||
metadata: None,
|
metadata: None,
|
||||||
|
replication_sse_headers: Vec::new(),
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -2985,6 +3350,8 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
Some(0),
|
Some(0),
|
||||||
|
ReplicationTimestampHeaders::default(),
|
||||||
|
ProxyHeaderSnapshot::default(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let records = lock(&control).requests.clone();
|
let records = lock(&control).requests.clone();
|
||||||
@@ -3006,6 +3373,8 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
None,
|
None,
|
||||||
|
ReplicationTimestampHeaders::default(),
|
||||||
|
ProxyHeaderSnapshot::default(),
|
||||||
);
|
);
|
||||||
{
|
{
|
||||||
let bounded_records = lock(&bounded_control);
|
let bounded_records = lock(&bounded_control);
|
||||||
|
|||||||
@@ -2610,17 +2610,20 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
|
|||||||
|
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
let payload: serde_json::Value = response.json().await?;
|
let payload: serde_json::Value = response.json().await?;
|
||||||
assert_eq!(payload["Status"], "OK");
|
assert_eq!(payload["Status"], "OK", "{payload}");
|
||||||
assert_eq!(payload["ActiveMutation"], true);
|
assert_eq!(payload["ActiveMutation"], true);
|
||||||
assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
|
assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
|
||||||
assert_eq!(payload["Targets"][0]["Status"], "OK");
|
assert_eq!(payload["Targets"][0]["Status"], "OK", "{payload}");
|
||||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK");
|
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||||
// A RustFS target adopts the source version id, so the P1-19
|
// A RustFS target adopts the source version id, so the P1-19
|
||||||
// version-identity probe passes.
|
// version-identity probe passes.
|
||||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK");
|
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK");
|
// A RustFS target preserves the SSE-C passthrough transport headers and
|
||||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK");
|
// echoes the customer algorithm on the replication-check HEAD (N2).
|
||||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK");
|
assert_eq!(payload["Targets"][0]["Phases"]["SsecPassthrough"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||||
|
|
||||||
let target_client = target_env.create_s3_client();
|
let target_client = target_env.create_s3_client();
|
||||||
let versions = target_client
|
let versions = target_client
|
||||||
@@ -4649,6 +4652,410 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// N2 (backlog#1675 P1-22): SSE-C passthrough replication to a target that
|
||||||
|
/// silently drops the `X-Rustfs-Replication-*` transport headers (MinIO-like
|
||||||
|
/// behavior, modeled by the fake target's drop mode) used to report COMPLETED
|
||||||
|
/// while the replica had irrecoverably lost its decryption material — the red
|
||||||
|
/// light this test was born failing on. Fail-closed contract now under test:
|
||||||
|
/// the first attempt PUTs, HEAD-backs the replica, finds no SSE-C evidence,
|
||||||
|
/// records the target Unsupported and reports FAILED; a second SSE-C object
|
||||||
|
/// fails without any PUT reaching the target (capability cache, proven from
|
||||||
|
/// the target journal); plaintext objects still replicate COMPLETED.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
let target_bucket = "ssec-drop-dst";
|
||||||
|
target.create_bucket(target_bucket);
|
||||||
|
target.drop_unlisted_replication_headers(true);
|
||||||
|
|
||||||
|
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||||
|
let mut env_vars = replication_fast_env();
|
||||||
|
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||||
|
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||||
|
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||||
|
|
||||||
|
let source_bucket = "ssec-drop-src";
|
||||||
|
let source_client = source_env.create_s3_client();
|
||||||
|
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||||
|
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||||
|
let target_arn = set_replication_target_with_options(
|
||||||
|
&source_env,
|
||||||
|
source_bucket,
|
||||||
|
ReplicationTargetOptions {
|
||||||
|
endpoint: &target.address(),
|
||||||
|
access_key: FAKE_ACCESS_KEY,
|
||||||
|
secret_key: FAKE_SECRET_KEY,
|
||||||
|
target_bucket,
|
||||||
|
secure: false,
|
||||||
|
skip_tls_verify: false,
|
||||||
|
ca_cert_pem: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||||
|
|
||||||
|
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||||
|
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||||
|
let put_ssec = |key: &'static str| {
|
||||||
|
source_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key(key)
|
||||||
|
.body(ByteStream::from_static(b"ssec fail-closed payload"))
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
};
|
||||||
|
|
||||||
|
// First SSE-C object: the audit must catch the dropped material.
|
||||||
|
put_ssec("ssec-first.txt").await?;
|
||||||
|
wait_for_source_replication_status(&source_client, source_bucket, "ssec-first.txt", "FAILED", true).await?;
|
||||||
|
|
||||||
|
let requests = target.take_requests();
|
||||||
|
let first_put = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::PutObject && record.key.as_deref() == Some("ssec-first.txt"))
|
||||||
|
.ok_or("the first SSE-C object must have been PUT (capability was Unknown)")?;
|
||||||
|
assert!(
|
||||||
|
first_put.proxy_headers.ssec_transport_present,
|
||||||
|
"the replication PUT must have shipped the SSE-C transport headers the target then dropped"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
requests.iter().any(|record| {
|
||||||
|
record.operation == FakeTargetOperation::HeadObject
|
||||||
|
&& record.key.as_deref() == Some("ssec-first.txt")
|
||||||
|
&& record.sequence > first_put.sequence
|
||||||
|
&& record.proxy_headers.replication_check.as_deref() == Some("true")
|
||||||
|
}),
|
||||||
|
"the post-PUT HEAD-back audit must have run through the replication-check channel; journal: {requests:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Second SSE-C object: the cached Unsupported verdict fails it closed
|
||||||
|
// before any PUT — including MRF retries of the first object.
|
||||||
|
put_ssec("ssec-second.txt").await?;
|
||||||
|
wait_for_source_replication_status(&source_client, source_bucket, "ssec-second.txt", "FAILED", true).await?;
|
||||||
|
assert!(
|
||||||
|
!target.requests().iter().any(|record| {
|
||||||
|
record.operation == FakeTargetOperation::PutObject
|
||||||
|
&& record.key.as_deref() != Some("plain-control.txt")
|
||||||
|
&& record.proxy_headers.ssec_transport_present
|
||||||
|
}),
|
||||||
|
"no further SSE-C ciphertext may reach a target recorded Unsupported; journal: {:?}",
|
||||||
|
target.requests()
|
||||||
|
);
|
||||||
|
|
||||||
|
// The gate is scoped to SSE-C: plaintext replication keeps working.
|
||||||
|
source_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("plain-control.txt")
|
||||||
|
.body(ByteStream::from_static(b"plaintext control payload"))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
wait_for_source_replication_status(&source_client, source_bucket, "plain-control.txt", "COMPLETED", false).await?;
|
||||||
|
assert!(target.has_object(target_bucket, "plain-control.txt"));
|
||||||
|
|
||||||
|
target.shutdown().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// N2 (backlog#1675 P1-22): the admin replication-check must expose the same
|
||||||
|
/// verdict operators would otherwise only learn from failing SSE-C objects —
|
||||||
|
/// an SsecPassthrough probe phase that fails with the machine-readable
|
||||||
|
/// `BucketRemoteSsecPassthroughUnsupported` code against a header-dropping
|
||||||
|
/// target, with no probe residue left behind. The target's overall status
|
||||||
|
/// stays OK: unlike version-identity drift, dropped passthrough headers are
|
||||||
|
/// a capability limit, and a plaintext-only deployment against a MinIO-like
|
||||||
|
/// target must not turn red.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
let target_bucket = "ssec-check-dst";
|
||||||
|
target.create_bucket(target_bucket);
|
||||||
|
target.drop_unlisted_replication_headers(true);
|
||||||
|
|
||||||
|
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||||
|
let mut env_vars = replication_fast_env();
|
||||||
|
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||||
|
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||||
|
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||||
|
|
||||||
|
let source_bucket = "ssec-check-src";
|
||||||
|
let source_client = source_env.create_s3_client();
|
||||||
|
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||||
|
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||||
|
let target_arn = set_replication_target_with_options(
|
||||||
|
&source_env,
|
||||||
|
source_bucket,
|
||||||
|
ReplicationTargetOptions {
|
||||||
|
endpoint: &target.address(),
|
||||||
|
access_key: FAKE_ACCESS_KEY,
|
||||||
|
secret_key: FAKE_SECRET_KEY,
|
||||||
|
target_bucket,
|
||||||
|
secure: false,
|
||||||
|
skip_tls_verify: false,
|
||||||
|
ca_cert_pem: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||||
|
|
||||||
|
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = response.json().await?;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
payload["Status"], "OK",
|
||||||
|
"a capability-only SSE-C failure must not fail the check overall: {payload}"
|
||||||
|
);
|
||||||
|
let target_report = &payload["Targets"][0];
|
||||||
|
assert_eq!(target_report["Status"], "OK", "{payload}");
|
||||||
|
let ssec = &target_report["Phases"]["SsecPassthrough"];
|
||||||
|
assert_eq!(ssec["Status"], "FAILED", "SsecPassthrough phase must fail: {payload}");
|
||||||
|
assert_eq!(
|
||||||
|
ssec["Code"], "BucketRemoteSsecPassthroughUnsupported",
|
||||||
|
"the failure must carry the machine-readable code: {payload}"
|
||||||
|
);
|
||||||
|
// Basic replication of plaintext objects works on this target: every other
|
||||||
|
// phase passes, so the code is the discriminator operators branch on.
|
||||||
|
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||||
|
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||||
|
|
||||||
|
// The SSE-C probe PUT must have shipped the real transport header names —
|
||||||
|
// a mangled or missing header set would fail the phase for the wrong
|
||||||
|
// reason and mask a working target.
|
||||||
|
let requests = target.requests();
|
||||||
|
assert!(
|
||||||
|
requests
|
||||||
|
.iter()
|
||||||
|
.any(|record| record.operation == FakeTargetOperation::PutObject && record.proxy_headers.ssec_transport_present),
|
||||||
|
"the SSE-C probe PUT must carry the X-Rustfs-Replication-* transport headers; journal: {requests:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// No probe residue, including the SSE-C probe version.
|
||||||
|
let probe_put = requests
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::PutObject)
|
||||||
|
.ok_or("the probe PUT never reached the fake target")?;
|
||||||
|
let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?;
|
||||||
|
assert!(
|
||||||
|
target.stored_versions(target_bucket, &probe_key).is_empty(),
|
||||||
|
"all probe versions must be cleaned up"
|
||||||
|
);
|
||||||
|
|
||||||
|
target.shutdown().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// C1 (backlog#1675 P1-22): heal-path convergence for SSE-C. An SSE-C object
|
||||||
|
/// whose live replication failed during a target outage must converge through
|
||||||
|
/// the scanner/heal compensation once the target returns — passing the N2
|
||||||
|
/// HEAD-back audit against the recovered RustFS target — and the replica must
|
||||||
|
/// be readable with the customer key.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let (source_env, mut target_env, source_bucket, target_bucket) =
|
||||||
|
build_sse_replication_pair("ssec-heal", false, false).await?;
|
||||||
|
let source_client = source_env.create_s3_client();
|
||||||
|
let key = "ssec-heal-contract.txt";
|
||||||
|
let body = b"repl-22 ssec heal payload".to_vec();
|
||||||
|
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||||
|
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||||
|
|
||||||
|
// Target outage: the SSE-C write cannot replicate.
|
||||||
|
target_env.stop_server();
|
||||||
|
|
||||||
|
source_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(&source_bucket)
|
||||||
|
.key(key)
|
||||||
|
.body(ByteStream::from(body.clone()))
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// The failure is observable on the source (SSE-C HEAD needs the key).
|
||||||
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||||
|
loop {
|
||||||
|
let head = source_client
|
||||||
|
.head_object()
|
||||||
|
.bucket(&source_bucket)
|
||||||
|
.key(key)
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
match head.replication_status().map(|status| status.as_str()) {
|
||||||
|
Some("PENDING") | Some("FAILED") => break,
|
||||||
|
other => {
|
||||||
|
if tokio::time::Instant::now() >= deadline {
|
||||||
|
return Err(format!("source SSE-C object never reported PENDING/FAILED; last status={other:?}").into());
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(200)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recover the target in place; the source scanner re-drives the failure.
|
||||||
|
target_env
|
||||||
|
.restart_server_preserving_data(vec![], &[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")])
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", true).await?;
|
||||||
|
|
||||||
|
// The healed replica is a REPLICA (status surfaces on HEAD) readable with
|
||||||
|
// the customer key.
|
||||||
|
let target_client = target_env.create_s3_client();
|
||||||
|
let replica_head = target_client
|
||||||
|
.head_object()
|
||||||
|
.bucket(&target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
assert_eq!(
|
||||||
|
replica_head.replication_status().map(|status| status.as_str()),
|
||||||
|
Some("REPLICA"),
|
||||||
|
"the healed copy must carry REPLICA status"
|
||||||
|
);
|
||||||
|
let replica = target_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(&target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||||
|
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// C1 (backlog#1675 P1-22): existing-object resync for SSE-C. An SSE-C object
|
||||||
|
/// written BEFORE any replication config must reach the RustFS target through
|
||||||
|
/// the existing-object resync (`replicate_all` transport, N2-audited), land as
|
||||||
|
/// a REPLICA, and read back with the customer key.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||||
|
let mut source_process_env = replication_fast_env();
|
||||||
|
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||||
|
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||||
|
source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||||
|
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||||
|
|
||||||
|
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||||
|
target_env
|
||||||
|
.start_rustfs_server_without_cleanup_with_env(&[
|
||||||
|
("NO_PROXY", "127.0.0.1,localhost"),
|
||||||
|
("HTTP_PROXY", ""),
|
||||||
|
("HTTPS_PROXY", ""),
|
||||||
|
])
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let source_bucket = "ssec-existing-src";
|
||||||
|
let target_bucket = "ssec-existing-dst";
|
||||||
|
let source_client = source_env.create_s3_client();
|
||||||
|
let target_client = target_env.create_s3_client();
|
||||||
|
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||||
|
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||||
|
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||||
|
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||||
|
|
||||||
|
// The SSE-C object exists before any replication wiring.
|
||||||
|
let key = "ssec-existing-contract.txt";
|
||||||
|
let body = b"repl-22 ssec existing-object payload".to_vec();
|
||||||
|
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||||
|
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||||
|
source_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key(key)
|
||||||
|
.body(ByteStream::from(body.clone()))
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// Wire replication (existing-object enabled) and drive a resync.
|
||||||
|
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||||
|
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||||
|
let (reset_arn, reset_id) = start_bucket_replication_reset(&source_env, source_bucket).await?;
|
||||||
|
assert_eq!(reset_arn, target_arn);
|
||||||
|
let terminal = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |status| {
|
||||||
|
status.reset_id == reset_id && matches!(status.status.as_str(), "Completed" | "Failed")
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
assert_eq!(terminal.status, "Completed", "SSE-C existing-object resync must complete");
|
||||||
|
assert!(terminal.replicated_count >= 1, "the existing SSE-C object must have been resynced");
|
||||||
|
|
||||||
|
// The replica is a REPLICA (status surfaces on HEAD) readable with the
|
||||||
|
// customer key.
|
||||||
|
let replica_head = target_client
|
||||||
|
.head_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
assert_eq!(
|
||||||
|
replica_head.replication_status().map(|status| status.as_str()),
|
||||||
|
Some("REPLICA"),
|
||||||
|
"the resynced copy must carry REPLICA status"
|
||||||
|
);
|
||||||
|
let replica = target_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||||
|
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||||
|
|
||||||
|
// No plaintext leak: the replica stays unreadable without the key.
|
||||||
|
assert!(
|
||||||
|
target_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key(key)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"SSE-C replica must not be readable without the customer key"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// backlog#1147 repl-17 / backlog#1783: SSE-S3 objects replicate by decrypting
|
/// backlog#1147 repl-17 / backlog#1783: SSE-S3 objects replicate by decrypting
|
||||||
/// at the source and re-encrypting on the target with the target's own KMS.
|
/// at the source and re-encrypting on the target with the target's own KMS.
|
||||||
/// The property backlog#1291 pinned — never a silent plaintext replica — still
|
/// The property backlog#1291 pinned — never a silent plaintext replica — still
|
||||||
@@ -8417,3 +8824,304 @@ async fn test_scanner_never_compensates_when_existing_object_replication_disable
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Shared setup for the P1-5 read-proxy scenarios (backlog#1675): a RustFS
|
||||||
|
/// source with an enabled replication rule pointing at the fake target, and
|
||||||
|
/// an object seeded DIRECTLY on the target — it exists remotely but not
|
||||||
|
/// locally, exactly the active-active replication-lag window the read proxy
|
||||||
|
/// serves.
|
||||||
|
async fn start_read_proxy_lab(
|
||||||
|
source_bucket: &str,
|
||||||
|
target_bucket: &str,
|
||||||
|
) -> Result<(FakeS3Target, RustFSTestEnvironment, Client, Client), Box<dyn Error + Send + Sync>> {
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
target.create_bucket(target_bucket);
|
||||||
|
target.assign_own_version_ids(true);
|
||||||
|
|
||||||
|
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||||
|
let mut process_env = replication_fast_env();
|
||||||
|
process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||||
|
process_env.extend_from_slice(&[
|
||||||
|
("NO_PROXY", "127.0.0.1,localhost"),
|
||||||
|
("HTTP_PROXY", ""),
|
||||||
|
("HTTPS_PROXY", ""),
|
||||||
|
("RUST_LOG", "error"),
|
||||||
|
]);
|
||||||
|
source_env.start_rustfs_server_with_env(vec![], &process_env).await?;
|
||||||
|
|
||||||
|
let source_client = source_env.create_s3_client();
|
||||||
|
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||||
|
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||||
|
let target_arn = set_replication_target_with_options(
|
||||||
|
&source_env,
|
||||||
|
source_bucket,
|
||||||
|
ReplicationTargetOptions {
|
||||||
|
endpoint: &target.address(),
|
||||||
|
access_key: FAKE_ACCESS_KEY,
|
||||||
|
secret_key: FAKE_SECRET_KEY,
|
||||||
|
target_bucket,
|
||||||
|
secure: false,
|
||||||
|
skip_tls_verify: false,
|
||||||
|
ca_cert_pem: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||||
|
|
||||||
|
let target_client = Client::from_conf(crate::common::build_test_s3_config(
|
||||||
|
target.endpoint(),
|
||||||
|
FAKE_ACCESS_KEY,
|
||||||
|
FAKE_SECRET_KEY,
|
||||||
|
None,
|
||||||
|
"read-proxy-e2e",
|
||||||
|
));
|
||||||
|
|
||||||
|
Ok((target, source_env, source_client, target_client))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// P1-5 (backlog#1675): during the active-active replication lag window a
|
||||||
|
/// GET/HEAD for an object the local site does not have yet is proxied to the
|
||||||
|
/// replication target. Pins the wire contract: the anti-loop
|
||||||
|
/// `source-proxy-request` marker is sent, the replication worker's
|
||||||
|
/// `source-replication-check` SSE-C exemption is NEVER sent, client SSE-C
|
||||||
|
/// headers are forwarded verbatim, and an inbound request that was itself
|
||||||
|
/// proxied is answered locally (404) without touching the target.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let source_bucket = "proxy-read-src";
|
||||||
|
let target_bucket = "proxy-read-dst";
|
||||||
|
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||||
|
|
||||||
|
let payload = b"proxy payload".to_vec();
|
||||||
|
target_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.body(ByteStream::from(payload.clone()))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
target.take_requests();
|
||||||
|
|
||||||
|
// a. GET of the locally-missing object is served through the proxy.
|
||||||
|
let got = source_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| format!("proxied GET failed: {}", err.into_service_error()))?;
|
||||||
|
assert_eq!(got.content_length, Some(payload.len() as i64));
|
||||||
|
let body = got.body.collect().await?.into_bytes();
|
||||||
|
assert_eq!(body.as_ref(), payload.as_slice(), "proxied GET must stream the target's body");
|
||||||
|
|
||||||
|
let get_record = target
|
||||||
|
.requests()
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||||
|
.ok_or("fake target never received the proxied GET")?;
|
||||||
|
assert_eq!(
|
||||||
|
get_record.proxy_headers.source_proxy_request.as_deref(),
|
||||||
|
Some("true"),
|
||||||
|
"proxied GET must carry the anti-loop source-proxy-request marker"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
get_record.proxy_headers.replication_check.is_none(),
|
||||||
|
"proxied GET must never carry the replication worker's source-replication-check exemption"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
get_record.proxy_headers.ssec_algorithm.is_none() && !get_record.proxy_headers.ssec_key_present,
|
||||||
|
"no client SSE-C headers were sent, so none may be forwarded"
|
||||||
|
);
|
||||||
|
|
||||||
|
// a2. Client SSE-C headers travel verbatim to the target (the target owns
|
||||||
|
// the real SSE-C decryption; the plaintext fake simply ignores them).
|
||||||
|
target.take_requests();
|
||||||
|
let ssec_key = "01234567890123456789012345678901";
|
||||||
|
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
|
||||||
|
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
|
||||||
|
let _ = source_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&ssec_key_b64)
|
||||||
|
.sse_customer_key_md5(&ssec_key_md5)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| format!("proxied SSE-C GET failed: {}", err.into_service_error()))?;
|
||||||
|
let ssec_record = target
|
||||||
|
.requests()
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||||
|
.ok_or("fake target never received the proxied SSE-C GET")?;
|
||||||
|
assert_eq!(ssec_record.proxy_headers.ssec_algorithm.as_deref(), Some("AES256"));
|
||||||
|
assert!(ssec_record.proxy_headers.ssec_key_present, "SSE-C key header must be forwarded verbatim");
|
||||||
|
assert_eq!(ssec_record.proxy_headers.ssec_key_md5.as_deref(), Some(ssec_key_md5.as_str()));
|
||||||
|
assert!(ssec_record.proxy_headers.replication_check.is_none());
|
||||||
|
|
||||||
|
// b. HEAD of the locally-missing object is served through the proxy.
|
||||||
|
target.take_requests();
|
||||||
|
let head = source_client
|
||||||
|
.head_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| format!("proxied HEAD failed: {}", err.into_service_error()))?;
|
||||||
|
assert_eq!(head.content_length, Some(payload.len() as i64));
|
||||||
|
let head_record = target
|
||||||
|
.requests()
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("proxy-only"))
|
||||||
|
.ok_or("fake target never received the proxied HEAD")?;
|
||||||
|
assert_eq!(head_record.proxy_headers.source_proxy_request.as_deref(), Some("true"));
|
||||||
|
assert!(head_record.proxy_headers.replication_check.is_none());
|
||||||
|
|
||||||
|
// c. Anti-loop: an inbound request that already carries the proxy marker
|
||||||
|
// is answered locally with 404 and never forwarded to the target.
|
||||||
|
target.take_requests();
|
||||||
|
let err = source_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.customize()
|
||||||
|
.mutate_request(|req| {
|
||||||
|
req.headers_mut().insert("x-minio-source-proxy-request", "true");
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect_err("anti-loop GET must fail locally instead of proxying");
|
||||||
|
let service_err = err.into_service_error();
|
||||||
|
assert!(service_err.is_no_such_key(), "anti-loop GET must 404, got: {service_err}");
|
||||||
|
assert!(
|
||||||
|
!target
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||||
|
"anti-loop GET must not reach the replication target; journal: {:?}",
|
||||||
|
target.requests()
|
||||||
|
);
|
||||||
|
|
||||||
|
// c2. MinIO ProxyHeaderSet parity: the header's mere PRESENCE disables
|
||||||
|
// proxying — "false" is exactly what a peer's replication worker sends on
|
||||||
|
// its convergence HEADs, and proxying that miss back would fake
|
||||||
|
// convergence.
|
||||||
|
target.take_requests();
|
||||||
|
let err = source_client
|
||||||
|
.get_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-only")
|
||||||
|
.customize()
|
||||||
|
.mutate_request(|req| {
|
||||||
|
req.headers_mut().insert("x-minio-source-proxy-request", "false");
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect_err("proxy-header-set GET must fail locally instead of proxying");
|
||||||
|
let service_err = err.into_service_error();
|
||||||
|
assert!(service_err.is_no_such_key(), "proxy-header-set GET must 404, got: {service_err}");
|
||||||
|
assert!(
|
||||||
|
!target
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||||
|
"proxy-header-set GET must not reach the replication target; journal: {:?}",
|
||||||
|
target.requests()
|
||||||
|
);
|
||||||
|
|
||||||
|
// d. The replication worker's own convergence HEAD against the target
|
||||||
|
// must carry `source-proxy-request: false` (never proxied back) and the
|
||||||
|
// replication-check exemption. Trigger real replication and inspect the
|
||||||
|
// fake journal.
|
||||||
|
target.take_requests();
|
||||||
|
source_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("worker-replicated")
|
||||||
|
.body(ByteStream::from_static(b"worker payload"))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "worker-replicated").await?;
|
||||||
|
let worker_head = target
|
||||||
|
.requests()
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("worker-replicated"))
|
||||||
|
.ok_or_else(|| format!("replication worker never HEAD-ed the target; journal: {:?}", target.requests()))?;
|
||||||
|
assert_eq!(
|
||||||
|
worker_head.proxy_headers.source_proxy_request.as_deref(),
|
||||||
|
Some("false"),
|
||||||
|
"worker convergence HEAD must send source-proxy-request: false so the target answers locally"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
worker_head.proxy_headers.replication_check.as_deref(),
|
||||||
|
Some("true"),
|
||||||
|
"worker convergence HEAD keeps the replication-check exemption"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(source_env);
|
||||||
|
target.shutdown().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// P1-5 (backlog#1675): GetObjectTagging for an object missing locally is
|
||||||
|
/// proxied to the replication target with the anti-loop marker, mirroring
|
||||||
|
/// MinIO `proxyGetTaggingToRepTarget`.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
|
||||||
|
let source_bucket = "proxy-tag-src";
|
||||||
|
let target_bucket = "proxy-tag-dst";
|
||||||
|
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||||
|
|
||||||
|
target_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key("proxy-tagged")
|
||||||
|
.body(ByteStream::from_static(b"tagged payload"))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
target_client
|
||||||
|
.put_object_tagging()
|
||||||
|
.bucket(target_bucket)
|
||||||
|
.key("proxy-tagged")
|
||||||
|
.tagging(
|
||||||
|
aws_sdk_s3::types::Tagging::builder()
|
||||||
|
.tag_set(aws_sdk_s3::types::Tag::builder().key("team").value("storage").build()?)
|
||||||
|
.build()?,
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
target.take_requests();
|
||||||
|
|
||||||
|
let tags = source_client
|
||||||
|
.get_object_tagging()
|
||||||
|
.bucket(source_bucket)
|
||||||
|
.key("proxy-tagged")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| format!("proxied GetObjectTagging failed: {}", err.into_service_error()))?;
|
||||||
|
assert_eq!(tags.tag_set.len(), 1, "proxied tagging read must return the target's tags");
|
||||||
|
assert_eq!(tags.tag_set[0].key.as_str(), "team");
|
||||||
|
assert_eq!(tags.tag_set[0].value.as_str(), "storage");
|
||||||
|
|
||||||
|
let record = target
|
||||||
|
.requests()
|
||||||
|
.into_iter()
|
||||||
|
.find(|record| record.operation == FakeTargetOperation::GetObjectTagging && record.key.as_deref() == Some("proxy-tagged"))
|
||||||
|
.ok_or("fake target never received the proxied GetObjectTagging")?;
|
||||||
|
assert_eq!(
|
||||||
|
record.proxy_headers.source_proxy_request.as_deref(),
|
||||||
|
Some("true"),
|
||||||
|
"proxied tagging read must carry the anti-loop marker"
|
||||||
|
);
|
||||||
|
assert!(record.proxy_headers.replication_check.is_none());
|
||||||
|
|
||||||
|
drop(source_env);
|
||||||
|
target.shutdown().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
|||||||
pub mod bucket_target_sys {
|
pub mod bucket_target_sys {
|
||||||
pub use crate::bucket::bucket_target_sys::{
|
pub use crate::bucket::bucket_target_sys::{
|
||||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||||
TargetClient, append_version_id_query,
|
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,7 +135,8 @@ pub mod bucket {
|
|||||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||||
pub use crate::bucket::metadata_sys::{
|
pub use crate::bucket::metadata_sys::{
|
||||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||||
|
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||||
@@ -184,24 +185,26 @@ pub mod bucket {
|
|||||||
mrf_backlog_observability_snapshot,
|
mrf_backlog_observability_snapshot,
|
||||||
};
|
};
|
||||||
pub use crate::bucket::replication::{
|
pub use crate::bucket::replication::{
|
||||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
||||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||||
|
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||||
|
version_purge_status_to_filemeta,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,14 +374,14 @@ pub mod error {
|
|||||||
|
|
||||||
pub mod erasure {
|
pub mod erasure {
|
||||||
pub use crate::erasure::coding::{
|
pub use crate::erasure::coding::{
|
||||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||||
calc_shard_size, calc_shard_size_legacy,
|
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod event {
|
pub mod event {
|
||||||
pub use crate::event::name::EventName;
|
pub use crate::event::name::EventName;
|
||||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod global {
|
pub mod global {
|
||||||
@@ -481,6 +484,7 @@ pub mod store_list {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub mod storage {
|
pub mod storage {
|
||||||
|
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||||
pub use crate::store::HealWalkVersion;
|
pub use crate::store::HealWalkVersion;
|
||||||
pub use crate::store::{
|
pub use crate::store::{
|
||||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||||
|
|||||||
@@ -27,10 +27,15 @@ use aws_sdk_s3::config::SharedHttpClient;
|
|||||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||||
use aws_sdk_s3::error::SdkError;
|
use aws_sdk_s3::error::SdkError;
|
||||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||||
|
use aws_sdk_s3::operation::delete_object_tagging::{DeleteObjectTaggingError, DeleteObjectTaggingOutput};
|
||||||
|
use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||||
|
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||||
|
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||||
use aws_sdk_s3::types::{
|
use aws_sdk_s3::types::{
|
||||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||||
};
|
};
|
||||||
@@ -57,8 +62,10 @@ use rustfs_utils::http::{
|
|||||||
is_rustfs_header, is_standard_header, is_storageclass_header,
|
is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||||
};
|
};
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||||
|
insert_header,
|
||||||
};
|
};
|
||||||
use rustls_pki_types::pem::PemObject;
|
use rustls_pki_types::pem::PemObject;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -80,7 +87,6 @@ use tracing::warn;
|
|||||||
use url::Url;
|
use url::Url;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
|
||||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||||
|
|
||||||
@@ -293,9 +299,41 @@ struct TargetClientBuildProbe {
|
|||||||
release: Arc<tokio::sync::Semaphore>,
|
release: Arc<tokio::sync::Semaphore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// SSE-C passthrough capability verdicts (see the enum's own docs in
|
||||||
|
/// `rustfs-replication`) are cached here per target ARN: entries follow the
|
||||||
|
/// `arn_remotes_map` lifecycle (rebuilding or removing a target resets its
|
||||||
|
/// capability to `Unknown`) and additionally expire after
|
||||||
|
/// [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], after which the next attempt
|
||||||
|
/// re-audits. Re-exported so existing `bucket_target_sys` consumers keep
|
||||||
|
/// their import path while the verdict vocabulary lives with the
|
||||||
|
/// replication decision logic.
|
||||||
|
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||||
|
|
||||||
|
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||||
|
///
|
||||||
|
/// Trade-off: without a TTL a verdict is sticky for the process lifetime —
|
||||||
|
/// an `Unsupported` target that gets upgraded (or re-probed only via
|
||||||
|
/// replication-check) would keep failing SSE-C replication forever, and the
|
||||||
|
/// fail-open twin: a `Supported` verdict would outlive a backend swapped
|
||||||
|
/// behind the same endpoint/ARN. With the TTL, a bad target costs at most
|
||||||
|
/// one wasted PUT+HEAD audit per TTL window, and a changed backend is
|
||||||
|
/// re-discovered within the same window.
|
||||||
|
pub const SSEC_PASSTHROUGH_CAPABILITY_TTL: Duration = Duration::from_secs(10 * 60);
|
||||||
|
|
||||||
|
/// A recorded SSE-C passthrough verdict plus when it was recorded, so reads
|
||||||
|
/// can report staleness against [`SSEC_PASSTHROUGH_CAPABILITY_TTL`].
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
struct SsecPassthroughRecord {
|
||||||
|
capability: SsecPassthroughCapability,
|
||||||
|
recorded_at: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct BucketTargetSys {
|
pub struct BucketTargetSys {
|
||||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||||
|
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||||
|
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||||
|
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||||
@@ -316,6 +354,7 @@ impl BucketTargetSys {
|
|||||||
fn new() -> Self {
|
fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||||
|
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||||
@@ -579,19 +618,59 @@ impl BucketTargetSys {
|
|||||||
let update_mutex = self.target_update_mutex(bucket).await;
|
let update_mutex = self.target_update_mutex(bucket).await;
|
||||||
let _update_guard = update_mutex.lock().await;
|
let _update_guard = update_mutex.lock().await;
|
||||||
|
|
||||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||||
|
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||||
|
// capability accessors).
|
||||||
let mut targets_map = self.targets_map.write().await;
|
let mut targets_map = self.targets_map.write().await;
|
||||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||||
let mut health_map = self.target_h_mutex.write().await;
|
let mut health_map = self.target_h_mutex.write().await;
|
||||||
|
|
||||||
if let Some(targets) = targets_map.remove(bucket) {
|
if let Some(targets) = targets_map.remove(bucket) {
|
||||||
|
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||||
for target in targets {
|
for target in targets {
|
||||||
arn_remotes_map.remove(&target.arn);
|
arn_remotes_map.remove(&target.arn);
|
||||||
health_map.remove(&target.arn);
|
health_map.remove(&target.arn);
|
||||||
|
ssec_map.remove(&target.arn);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||||
|
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||||
|
/// false)` when no verdict has been recorded since the target was built.
|
||||||
|
/// Staleness is computed here so the gate policy stays a pure function.
|
||||||
|
pub async fn ssec_passthrough_capability(&self, arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||||
|
match self.ssec_passthrough_map.read().await.get(arn) {
|
||||||
|
Some(record) => (record.capability, record.recorded_at.elapsed() >= SSEC_PASSTHROUGH_CAPABILITY_TTL),
|
||||||
|
None => (SsecPassthroughCapability::Unknown, false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record an audited SSE-C passthrough verdict for a target ARN. Written by
|
||||||
|
/// the replication worker's HEAD-back audit and by the replication-check
|
||||||
|
/// SsecPassthrough probe phase.
|
||||||
|
pub async fn record_ssec_passthrough_capability(&self, arn: &str, capability: SsecPassthroughCapability) {
|
||||||
|
self.ssec_passthrough_map.write().await.insert(
|
||||||
|
arn.to_string(),
|
||||||
|
SsecPassthroughRecord {
|
||||||
|
capability,
|
||||||
|
recorded_at: Instant::now(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test hook: age an existing verdict so TTL expiry is observable without
|
||||||
|
/// waiting out the real window.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) async fn backdate_ssec_passthrough_capability(&self, arn: &str, age: Duration) {
|
||||||
|
let backdated = Instant::now()
|
||||||
|
.checked_sub(age)
|
||||||
|
.expect("system uptime must exceed the backdate age");
|
||||||
|
if let Some(record) = self.ssec_passthrough_map.write().await.get_mut(arn) {
|
||||||
|
record.recorded_at = backdated;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn set_target(
|
pub async fn set_target(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -947,15 +1026,21 @@ impl BucketTargetSys {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||||
|
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||||
|
// capability accessors).
|
||||||
let mut targets_map = self.targets_map.write().await;
|
let mut targets_map = self.targets_map.write().await;
|
||||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||||
let mut health_map = self.target_h_mutex.write().await;
|
let mut health_map = self.target_h_mutex.write().await;
|
||||||
// Remove existing targets
|
// Remove existing targets
|
||||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||||
|
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||||
for target in existing_targets {
|
for target in existing_targets {
|
||||||
arn_remotes_map.remove(&target.arn);
|
arn_remotes_map.remove(&target.arn);
|
||||||
health_map.remove(&target.arn);
|
health_map.remove(&target.arn);
|
||||||
|
// A rebuilt/edited target may point at a different service:
|
||||||
|
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||||
|
ssec_map.remove(&target.arn);
|
||||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1445,6 +1530,43 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolve the S3 `versionId` for a proxied read against a remote target.
|
||||||
|
/// RustFS represents the null version internally as the nil UUID while the S3
|
||||||
|
/// API addresses it as the literal "null" (same mapping as
|
||||||
|
/// [`resolve_put_api_version_id`]); empty means "no version requested".
|
||||||
|
pub(crate) fn resolve_read_api_version_id(version_id: Option<String>) -> Option<String> {
|
||||||
|
let version_id = version_id?;
|
||||||
|
let trimmed = version_id.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
None
|
||||||
|
} else if Uuid::parse_str(trimmed).is_ok_and(|uuid| uuid.is_nil()) {
|
||||||
|
Some(rustfs_filemeta::NULL_VERSION_ID.to_string())
|
||||||
|
} else {
|
||||||
|
Some(trimmed.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Outbound header set for a proxied read: the caller-provided passthrough
|
||||||
|
/// headers (client SSE-C key family, conditional headers) plus the anti-loop
|
||||||
|
/// `source-proxy-request` marker in both the x-rustfs- and x-minio- prefixes
|
||||||
|
/// (a MinIO target only understands the latter). Never adds
|
||||||
|
/// `source-replication-check`: that exemption channel belongs exclusively to
|
||||||
|
/// the replication worker's HEAD.
|
||||||
|
fn proxy_outbound_headers(mut extra_headers: HeaderMap) -> HeaderMap {
|
||||||
|
insert_header(&mut extra_headers, SUFFIX_SOURCE_PROXY_REQUEST, "true");
|
||||||
|
extra_headers
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy `headers` onto an SDK request inside `customize().map_request` (runs
|
||||||
|
/// before signing, so the headers join the SigV4 canonical request).
|
||||||
|
fn apply_extra_headers(mut req: HttpRequest, headers: &HeaderMap) -> Result<HttpRequest, std::convert::Infallible> {
|
||||||
|
for (k, v) in headers.iter() {
|
||||||
|
req.headers_mut()
|
||||||
|
.insert(k.as_str().to_string(), v.to_str().unwrap_or("").to_string());
|
||||||
|
}
|
||||||
|
Ok(req)
|
||||||
|
}
|
||||||
|
|
||||||
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
||||||
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
||||||
/// member, so the query is spliced in via `map_request`, which runs at
|
/// member, so the query is spliced in via `map_request`, which runs at
|
||||||
@@ -1476,9 +1598,12 @@ impl Default for AdvancedPutOptions {
|
|||||||
replication_status: ReplicationStatusType::Pending,
|
replication_status: ReplicationStatusType::Pending,
|
||||||
source_mtime: OffsetDateTime::now_utc(),
|
source_mtime: OffsetDateTime::now_utc(),
|
||||||
replication_request: false,
|
replication_request: false,
|
||||||
retention_timestamp: OffsetDateTime::now_utc(),
|
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
// timestamp header for it, otherwise a receiver would treat an
|
||||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
// unset category as a modification made right now.
|
||||||
|
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
|
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
|
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
replication_validity_check: false,
|
replication_validity_check: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1545,8 +1670,8 @@ impl Default for PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl PutObjectOptions {
|
impl PutObjectOptions {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn set_match_etag(&mut self, etag: &str) {
|
fn set_match_etag(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header
|
self.custom_header
|
||||||
@@ -1557,6 +1682,7 @@ impl PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn set_match_etag_except(&mut self, etag: &str) {
|
fn set_match_etag_except(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header
|
self.custom_header
|
||||||
@@ -1675,6 +1801,16 @@ impl PutObjectOptions {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (suffix, timestamp) in [
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||||
|
] {
|
||||||
|
if timestamp.unix_timestamp() != 0 {
|
||||||
|
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if self.internal.replication_request {
|
if self.internal.replication_request {
|
||||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||||
}
|
}
|
||||||
@@ -1682,6 +1818,7 @@ impl PutObjectOptions {
|
|||||||
header
|
header
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||||
//if self.checksum.is_set() {
|
//if self.checksum.is_set() {
|
||||||
/*if !self.trailing_header_support {
|
/*if !self.trailing_header_support {
|
||||||
@@ -1837,6 +1974,13 @@ impl TargetClient {
|
|||||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||||
let mut headers = HeaderMap::new();
|
let mut headers = HeaderMap::new();
|
||||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||||
|
// `source-proxy-request: false` (MinIO `ProxyHeaderSet` semantics):
|
||||||
|
// the header's mere presence tells the receiver to answer LOCALLY
|
||||||
|
// instead of proxying the miss back to us. Without it, a not-found on
|
||||||
|
// the target gets read-proxied back to this source, echoes the source
|
||||||
|
// object with an identical ETag, and the worker concludes the object
|
||||||
|
// already converged — so it never actually replicates it.
|
||||||
|
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||||
match self
|
match self
|
||||||
.client
|
.client
|
||||||
.head_object()
|
.head_object()
|
||||||
@@ -1861,6 +2005,129 @@ impl TargetClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||||
|
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||||
|
///
|
||||||
|
/// Deliberately different from [`TargetClient::head_object`]: it must NOT
|
||||||
|
/// send `source-replication-check` — that header is the replication
|
||||||
|
/// worker's SSE-C metadata exemption channel. A proxied client request
|
||||||
|
/// instead forwards the client's own SSE-C headers (`extra_headers`) so
|
||||||
|
/// the target performs the real SSE-C validation/decryption. The
|
||||||
|
/// `source-proxy-request` marker is always added so the target does not
|
||||||
|
/// proxy the request onward (anti-loop).
|
||||||
|
pub async fn head_object_for_proxy(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<String>,
|
||||||
|
range: Option<String>,
|
||||||
|
part_number: Option<i32>,
|
||||||
|
extra_headers: HeaderMap,
|
||||||
|
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||||
|
let headers = proxy_outbound_headers(extra_headers);
|
||||||
|
self.client
|
||||||
|
.head_object()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.set_version_id(resolve_read_api_version_id(version_id))
|
||||||
|
.set_range(range)
|
||||||
|
.set_part_number(part_number)
|
||||||
|
.customize()
|
||||||
|
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||||
|
/// Returns the streaming SDK output; callers must forward the body without
|
||||||
|
/// buffering it. Same header contract as [`Self::head_object_for_proxy`]:
|
||||||
|
/// anti-loop marker on, replication-check never sent, client SSE-C /
|
||||||
|
/// conditional headers forwarded verbatim via `extra_headers`.
|
||||||
|
pub async fn get_object(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<String>,
|
||||||
|
range: Option<String>,
|
||||||
|
part_number: Option<i32>,
|
||||||
|
extra_headers: HeaderMap,
|
||||||
|
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||||
|
let headers = proxy_outbound_headers(extra_headers);
|
||||||
|
self.client
|
||||||
|
.get_object()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.set_version_id(resolve_read_api_version_id(version_id))
|
||||||
|
.set_range(range)
|
||||||
|
.set_part_number(part_number)
|
||||||
|
.customize()
|
||||||
|
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GetObjectTagging for the tagging read-proxy path
|
||||||
|
/// (MinIO `proxyGetTaggingToRepTarget`). Anti-loop marker always added.
|
||||||
|
pub async fn get_object_tagging(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<String>,
|
||||||
|
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||||
|
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||||
|
self.client
|
||||||
|
.get_object_tagging()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.set_version_id(resolve_read_api_version_id(version_id))
|
||||||
|
.customize()
|
||||||
|
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// PutObjectTagging for the tagging proxy path
|
||||||
|
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||||
|
pub async fn put_object_tagging(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<String>,
|
||||||
|
tagging: SdkTagging,
|
||||||
|
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||||
|
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||||
|
self.client
|
||||||
|
.put_object_tagging()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.set_version_id(resolve_read_api_version_id(version_id))
|
||||||
|
.tagging(tagging)
|
||||||
|
.customize()
|
||||||
|
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DeleteObjectTagging for the tagging proxy path
|
||||||
|
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||||
|
pub async fn delete_object_tagging(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<String>,
|
||||||
|
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||||
|
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||||
|
self.client
|
||||||
|
.delete_object_tagging()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.set_version_id(resolve_read_api_version_id(version_id))
|
||||||
|
.customize()
|
||||||
|
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
/// On success returns the version id the target assigned (from
|
/// On success returns the version id the target assigned (from
|
||||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||||
/// contract — a target that adopts the source version echoes it back.
|
/// contract — a target that adopts the source version echoes it back.
|
||||||
@@ -2490,6 +2757,57 @@ mod tests {
|
|||||||
assert_eq!(health.last_online, Some(now));
|
assert_eq!(health.last_online, Some(now));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// N2 TTL contract, both flip directions: a recorded verdict is fresh
|
||||||
|
/// until [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], then reads as expired; a
|
||||||
|
/// re-audit that records the OPPOSITE verdict replaces it as fresh. The
|
||||||
|
/// worker gate maps expired verdicts to ProceedWithAudit (pinned in
|
||||||
|
/// `replication_target_boundary`), so together this proves an Unsupported
|
||||||
|
/// target recovers to Supported through the audit once its verdict ages
|
||||||
|
/// out — and a stale Supported one is re-proven rather than trusted.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ssec_passthrough_capability_ttl_expires_and_reaudit_flips_verdict() {
|
||||||
|
let sys = BucketTargetSys::default();
|
||||||
|
let arn = "arn:rustfs:replication:us-east-1:bucket:ssec-ttl";
|
||||||
|
let expired_age = SSEC_PASSTHROUGH_CAPABILITY_TTL + Duration::from_secs(1);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sys.ssec_passthrough_capability(arn).await,
|
||||||
|
(SsecPassthroughCapability::Unknown, false),
|
||||||
|
"an unrecorded target must read Unknown and never expired"
|
||||||
|
);
|
||||||
|
|
||||||
|
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Unsupported)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
sys.ssec_passthrough_capability(arn).await,
|
||||||
|
(SsecPassthroughCapability::Unsupported, false)
|
||||||
|
);
|
||||||
|
|
||||||
|
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||||
|
assert_eq!(
|
||||||
|
sys.ssec_passthrough_capability(arn).await,
|
||||||
|
(SsecPassthroughCapability::Unsupported, true),
|
||||||
|
"an aged-out Unsupported verdict must read expired so the gate re-audits"
|
||||||
|
);
|
||||||
|
|
||||||
|
// The re-audit against an upgraded target records Supported afresh.
|
||||||
|
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Supported)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
sys.ssec_passthrough_capability(arn).await,
|
||||||
|
(SsecPassthroughCapability::Supported, false),
|
||||||
|
"a fresh Supported verdict replaces the expired Unsupported one"
|
||||||
|
);
|
||||||
|
|
||||||
|
// And the fail-open twin: Supported also ages out.
|
||||||
|
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||||
|
assert_eq!(
|
||||||
|
sys.ssec_passthrough_capability(arn).await,
|
||||||
|
(SsecPassthroughCapability::Supported, true),
|
||||||
|
"an aged-out Supported verdict must read expired so the gate re-proves it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
||||||
let sys = BucketTargetSys::default();
|
let sys = BucketTargetSys::default();
|
||||||
@@ -2842,6 +3160,57 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||||
|
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||||
|
// last-writer-wins on these headers (object-api-options.go parses them
|
||||||
|
// as RFC3339); a replica without them loses every conflict resolution.
|
||||||
|
let mut opts = PutObjectOptions::default();
|
||||||
|
opts.internal.replication_request = true;
|
||||||
|
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||||
|
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||||
|
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||||
|
opts.internal.tagging_timestamp = tagging;
|
||||||
|
opts.internal.retention_timestamp = retention;
|
||||||
|
opts.internal.legalhold_timestamp = legalhold;
|
||||||
|
|
||||||
|
let header = opts.header();
|
||||||
|
for (suffix, expected) in [
|
||||||
|
("source-replication-tagging-timestamp", tagging),
|
||||||
|
("source-replication-retention-timestamp", retention),
|
||||||
|
("source-replication-legalhold-timestamp", legalhold),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||||
|
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||||
|
"replication put requests must carry the {suffix} header"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||||
|
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||||
|
// make the receiver treat an unset category as a fresh modification.
|
||||||
|
let mut opts = PutObjectOptions::default();
|
||||||
|
opts.internal.replication_request = true;
|
||||||
|
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
|
||||||
|
let header = opts.header();
|
||||||
|
for suffix in [
|
||||||
|
"source-replication-tagging-timestamp",
|
||||||
|
"source-replication-retention-timestamp",
|
||||||
|
"source-replication-legalhold-timestamp",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||||
|
"unset {suffix} must not be sent to replication targets"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||||
let sys = BucketTargetSys::default();
|
let sys = BucketTargetSys::default();
|
||||||
|
|||||||
@@ -126,11 +126,23 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
|
|||||||
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
||||||
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
||||||
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
||||||
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
||||||
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub type TraceFn =
|
pub type TraceFn =
|
||||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||||
@@ -140,9 +152,21 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
|||||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||||
|
|
||||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
||||||
|
|
||||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||||
@@ -280,6 +304,10 @@ impl LifecycleSys {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||||
let bucket = oi.bucket.clone();
|
let bucket = oi.bucket.clone();
|
||||||
let name = oi.name.clone();
|
let name = oi.name.clone();
|
||||||
@@ -428,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl ExpiryStats {
|
impl ExpiryStats {
|
||||||
pub fn missed_tasks(&self) -> i64 {
|
pub fn missed_tasks(&self) -> i64 {
|
||||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||||
|
)]
|
||||||
fn missed_free_vers_tasks(&self) -> i64 {
|
fn missed_free_vers_tasks(&self) -> i64 {
|
||||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||||
|
)]
|
||||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
@@ -570,6 +605,10 @@ async fn delete_free_version_remote_object(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
||||||
oi: &ObjectInfo,
|
oi: &ObjectInfo,
|
||||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||||
@@ -1744,7 +1783,7 @@ impl TransitionState {
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
global_metrics().record_scanner_transition_failed(1);
|
global_metrics().record_scanner_transition_failed(1);
|
||||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
|
||||||
error!(
|
error!(
|
||||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
@@ -2868,6 +2907,10 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
|||||||
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||||
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||||
}
|
}
|
||||||
@@ -3352,6 +3395,10 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
||||||
if remote_delete_succeeded {
|
if remote_delete_succeeded {
|
||||||
opts.skip_decommissioned = true;
|
opts.skip_decommissioned = true;
|
||||||
@@ -4339,6 +4386,10 @@ pub async fn expire_transitioned_object(
|
|||||||
Ok(dobj)
|
Ok(dobj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||||
let us = Uuid::new_v4().to_string();
|
let us = Uuid::new_v4().to_string();
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
@@ -4373,6 +4424,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||||
let tier = _tier.to_string();
|
let tier = _tier.to_string();
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
@@ -4391,6 +4446,10 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn get_transitioned_object_reader(
|
pub async fn get_transitioned_object_reader(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
@@ -5145,6 +5204,10 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
|
|||||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||||
let mut success = false;
|
let mut success = false;
|
||||||
match event.action {
|
match event.action {
|
||||||
@@ -7422,6 +7485,10 @@ mod tests {
|
|||||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||||
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
||||||
#[allow(unsafe_code)]
|
#[allow(unsafe_code)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
||||||
where
|
where
|
||||||
F: FnOnce() -> Fut,
|
F: FnOnce() -> Fut,
|
||||||
|
|||||||
@@ -759,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ManualTransitionWorkerResultRecord {
|
impl ManualTransitionWorkerResultRecord {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||||
Self::new_with_reason(job_id, task_key, result, None)
|
Self::new_with_reason(job_id, task_key, result, None)
|
||||||
}
|
}
|
||||||
@@ -1257,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn load_manual_transition_task_record(
|
pub async fn load_manual_transition_task_record(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1320,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn load_manual_transition_worker_result_stats(
|
pub async fn load_manual_transition_worker_result_stats(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1455,6 +1467,10 @@ async fn scan_manual_transition_worker_result_journal(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn reconcile_manual_transition_worker_results(
|
pub async fn reconcile_manual_transition_worker_results(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ pub mod core;
|
|||||||
pub mod evaluator;
|
pub mod evaluator;
|
||||||
pub mod manual_transition_job;
|
pub mod manual_transition_job;
|
||||||
mod metadata_boundary;
|
mod metadata_boundary;
|
||||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||||
mod object_lock_boundary;
|
mod object_lock_boundary;
|
||||||
pub use self::core as lifecycle;
|
pub use self::core as lifecycle;
|
||||||
mod replication_sink;
|
mod replication_sink;
|
||||||
|
|||||||
@@ -15,25 +15,35 @@
|
|||||||
use rustfs_common::metrics::IlmAction;
|
use rustfs_common::metrics::IlmAction;
|
||||||
|
|
||||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||||
|
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||||
pub(crate) use crate::bucket::replication::{
|
pub(crate) use crate::bucket::replication::{
|
||||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||||
};
|
};
|
||||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
|
||||||
use crate::storage_api_contracts::object::DeletedObject;
|
use crate::storage_api_contracts::object::DeletedObject;
|
||||||
|
|
||||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||||
obj.version_purge_status.is_pending()
|
obj.version_purge_status.is_pending()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,10 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -331,6 +331,10 @@ where
|
|||||||
persist_tier_delete_journal_entry(api, &committed).await
|
persist_tier_delete_journal_entry(api, &committed).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||||
where
|
where
|
||||||
S: ObjectOperations<
|
S: ObjectOperations<
|
||||||
|
|||||||
@@ -148,6 +148,10 @@ struct RecoveryCursor {
|
|||||||
object: String,
|
object: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn recover_tier_free_versions(
|
pub async fn recover_tier_free_versions(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
|
|||||||
@@ -80,7 +80,10 @@ impl LastDayTierStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||||
|
)]
|
||||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||||
let mut cl = self.clone();
|
let mut cl = self.clone();
|
||||||
let mut cm = m;
|
let mut cm = m;
|
||||||
|
|||||||
@@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
#[allow(dead_code)]
|
|
||||||
struct ObjSweeper {
|
struct ObjSweeper {
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
object: String,
|
object: String,
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
bucket: String,
|
bucket: String,
|
||||||
version_id: Option<Uuid>,
|
version_id: Option<Uuid>,
|
||||||
versioned: bool,
|
versioned: bool,
|
||||||
@@ -191,9 +192,9 @@ struct ObjSweeper {
|
|||||||
remote_object: String,
|
remote_object: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl ObjSweeper {
|
impl ObjSweeper {
|
||||||
#[allow(clippy::new_ret_no_self)]
|
#[allow(clippy::new_ret_no_self)]
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
object: object.into(),
|
object: object.into(),
|
||||||
@@ -202,17 +203,20 @@ impl ObjSweeper {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||||
self.version_id = vid.clone();
|
self.version_id = vid.clone();
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||||
self.versioned = versioned;
|
self.versioned = versioned;
|
||||||
self.suspended = suspended;
|
self.suspended = suspended;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||||
let mut opts = ObjectOpts {
|
let mut opts = ObjectOpts {
|
||||||
version_id: self.version_id.clone(),
|
version_id: self.version_id.clone(),
|
||||||
@@ -226,6 +230,7 @@ impl ObjSweeper {
|
|||||||
opts
|
opts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||||
self.transition_tier = info.tier;
|
self.transition_tier = info.tier;
|
||||||
self.transition_status = info.status;
|
self.transition_status = info.status;
|
||||||
@@ -266,6 +271,7 @@ impl ObjSweeper {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||||
let Some(je) = self.should_remove_remote_object() else {
|
let Some(je) = self.should_remove_remote_object() else {
|
||||||
return;
|
return;
|
||||||
@@ -385,6 +391,10 @@ impl ExpiryOp for Jentry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||||
if let Err(err) = &result
|
if let Err(err) = &result
|
||||||
@@ -395,6 +405,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||||
@@ -405,6 +419,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
|||||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
@@ -485,6 +503,10 @@ pub enum RemoteTierDeleteOutcome {
|
|||||||
AlreadyRemoved,
|
AlreadyRemoved,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_object_from_remote_tier_idempotent(
|
pub async fn delete_object_from_remote_tier_idempotent(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
|
|||||||
@@ -50,8 +50,16 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
|||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum TransitionTransactionError {
|
pub enum TransitionTransactionError {
|
||||||
#[error("transition transaction already exists")]
|
#[error("transition transaction already exists")]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
AlreadyExists,
|
AlreadyExists,
|
||||||
#[error("transition transaction is not found")]
|
#[error("transition transaction is not found")]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
NotFound,
|
NotFound,
|
||||||
#[error("transition transaction is corrupt: {0}")]
|
#[error("transition transaction is corrupt: {0}")]
|
||||||
Corrupt(&'static str),
|
Corrupt(&'static str),
|
||||||
|
|||||||
@@ -60,12 +60,14 @@ struct ConfigWriteLockProbeState {
|
|||||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub struct ConfigWriteLockProbe {
|
pub struct ConfigWriteLockProbe {
|
||||||
state: Arc<ConfigWriteLockProbeState>,
|
state: Arc<ConfigWriteLockProbeState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
impl ConfigWriteLockProbe {
|
impl ConfigWriteLockProbe {
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub fn install(bucket: &str) -> Self {
|
pub fn install(bucket: &str) -> Self {
|
||||||
let state = Arc::new(ConfigWriteLockProbeState {
|
let state = Arc::new(ConfigWriteLockProbeState {
|
||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
@@ -84,6 +86,7 @@ impl ConfigWriteLockProbe {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub async fn wait_until_attempted(&self) {
|
pub async fn wait_until_attempted(&self) {
|
||||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
.await
|
.await
|
||||||
@@ -656,6 +659,16 @@ pub async fn update_under_transaction_lock(
|
|||||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||||
|
pub async fn delete_under_transaction_lock(
|
||||||
|
guard: &BucketMetadataMutationGuard,
|
||||||
|
bucket: &str,
|
||||||
|
config_file: &str,
|
||||||
|
) -> Result<OffsetDateTime> {
|
||||||
|
guard.ensure_valid(bucket)?;
|
||||||
|
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn update_quota_if_incarnation(
|
pub async fn update_quota_if_incarnation(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
data: Vec<u8>,
|
data: Vec<u8>,
|
||||||
@@ -795,6 +808,14 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
|||||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
||||||
|
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
||||||
|
bucket: &str,
|
||||||
|
expected_incarnation_id: Uuid,
|
||||||
|
) -> Result<BucketMetadataMutationGuard> {
|
||||||
|
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||||
ctx: &crate::runtime::instance::InstanceContext,
|
ctx: &crate::runtime::instance::InstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -872,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
|||||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1086,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
|||||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1599,6 +1628,7 @@ impl BucketMetadataSys {
|
|||||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||||
/// instead of supplied up front. Loads through this system's own store so
|
/// instead of supplied up front. Loads through this system's own store so
|
||||||
/// the read and the persisted write target the same instance.
|
/// the read and the persisted write target the same instance.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||||
where
|
where
|
||||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||||
@@ -1703,6 +1733,7 @@ impl BucketMetadataSys {
|
|||||||
/// A miss is never published as an authoritative default, and a snapshot
|
/// A miss is never published as an authoritative default, and a snapshot
|
||||||
/// read before delete plus same-name recreation cannot replace the new
|
/// read before delete plus same-name recreation cannot replace the new
|
||||||
/// generation.
|
/// generation.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||||
if is_meta_bucketname(bucket) {
|
if is_meta_bucketname(bucket) {
|
||||||
return Err(Error::other("errInvalidArgument"));
|
return Err(Error::other("errInvalidArgument"));
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod bandwidth;
|
pub mod bandwidth;
|
||||||
pub mod bucket_target_sys;
|
pub mod bucket_target_sys;
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
|||||||
|
|
||||||
/// Check if an object has legal hold enabled.
|
/// Check if an object has legal hold enabled.
|
||||||
/// Returns true if legal hold is ON.
|
/// Returns true if legal hold is ON.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||||
@@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||||
/// * `false` if the object is not locked
|
/// * `false` if the object is not locked
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||||
// Delete markers are never locked
|
// Delete markers are never locked
|
||||||
if is_delete_marker {
|
if is_delete_marker {
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ pub enum QuotaError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub struct QuotaErrorResponse {
|
pub struct QuotaErrorResponse {
|
||||||
#[serde(rename = "Code")]
|
#[serde(rename = "Code")]
|
||||||
pub code: String,
|
pub code: String,
|
||||||
@@ -208,6 +209,7 @@ pub struct QuotaErrorResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl QuotaErrorResponse {
|
impl QuotaErrorResponse {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||||
match quota_error {
|
match quota_error {
|
||||||
QuotaError::QuotaExceeded { .. } => Self {
|
QuotaError::QuotaExceeded { .. } => Self {
|
||||||
@@ -310,9 +312,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct LegacyBucketQuota {
|
struct LegacyBucketQuota {
|
||||||
#[allow(dead_code)]
|
|
||||||
quota: Option<u64>,
|
quota: Option<u64>,
|
||||||
#[allow(dead_code)]
|
|
||||||
quota_type: LegacyQuotaType,
|
quota_type: LegacyQuotaType,
|
||||||
}
|
}
|
||||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||||
|
|||||||
@@ -899,6 +899,7 @@ async fn save_ledger_locked(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn fail_next_quota_ledger_save_for_test() {
|
pub fn fail_next_quota_ledger_save_for_test() {
|
||||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ paths.
|
|||||||
| Module | Current role | Split blocker |
|
| Module | Current role | Split blocker |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local filemeta/tagging boundaries and S3 DTOs directly. |
|
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local filemeta/tagging boundaries and S3 DTOs directly. |
|
||||||
| `datatypes.rs` | ECStore compatibility re-export for resync status enums. | Re-exports `rustfs-replication` contracts while downstream facade consumers migrate. |
|
|
||||||
| `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. |
|
| `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. |
|
||||||
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
|
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
|
||||||
|
| `replication_proxy.rs` | Proxy-target selection for GET/HEAD/Tagging reads of objects not yet replicated locally (MinIO `getProxyTargets` parity: anti-loop, version-suspended, and no-config empty branches). | Uses replication config lookup, rule matching, and target clients through local boundaries. |
|
||||||
| `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. |
|
| `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. |
|
||||||
| `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. |
|
| `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. |
|
||||||
| `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
|
| `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
|
||||||
@@ -117,9 +117,12 @@ Target end state:
|
|||||||
their file names — so batch-merging them beforehand is explicitly rejected:
|
their file names — so batch-merging them beforehand is explicitly rejected:
|
||||||
it forces synchronized guard-script/mod/import churn with zero functional
|
it forces synchronized guard-script/mod/import churn with zero functional
|
||||||
gain;
|
gain;
|
||||||
- the only module that can retire early is `datatypes.rs`: delete it once its
|
- `datatypes.rs` retired early (its sanctioned exception): it was a pure
|
||||||
facade consumers import the resync status enums through `rustfs-replication`
|
relay (`boundary -> datatypes -> mod.rs`), so the facade now re-exports
|
||||||
directly.
|
`ResyncStatusType` from the resync boundary directly and the relay file is
|
||||||
|
deleted. Note the original retirement wording ("consumers import through
|
||||||
|
`rustfs-replication` directly") conflicted with Migration Rule #15 —
|
||||||
|
consumers stay behind the ECStore facade; only the relay hop dissolves.
|
||||||
|
|
||||||
## Milestones
|
## Milestones
|
||||||
|
|
||||||
@@ -127,9 +130,9 @@ Target end state:
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| M0 | Record the completion criteria and end state (this section). | Done |
|
| M0 | Record the completion criteria and end state (this section). | Done |
|
||||||
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
||||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
|
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Done — moved the pure decision helpers with their unit tests: `resync_status_duration` (resync), `resync_existing_delete_replication_info` / `replicate_delete_outcome` / `target_delete_version_id` / `delete_marker_purge_version_id` / `delete_marker_purge_mrf_entry` (delete), `version_identity_drifted` / `is_replication_target_offline_error` / the SSE-C passthrough gate family incl. `SsecPassthroughCapability` (object; `ssec_passthrough_evidence_present` was param-demoted to the echoed customer-algorithm string, ECStore keeps the `HeadObjectOutput` adapter). ECStore imports them through the resync/object-decision/target boundaries; `bucket_target_sys` keeps only the verdict cache + TTL and re-exports the capability enum. Not moved (signatures carry ECStore or aws-sdk types): `verify_resync_head_result`, `resync_target_error_detail`, the `SdkError` classifiers (`has_raw_status`, `is_version_id_format_mismatch`), the `replicate_all_*` option/info builders, and `bounded_resync_max_jobs` (itself a pure clamp, but it forms one local configuration unit with the env-reading `configured_resync_max_jobs` and its ECStore-local constants — moving the clamp alone has negative value). |
|
||||||
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
||||||
| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
|
| M4 | Retire the boundary modules together with their guard-script entries. | Pending (`datatypes.rs` already retired early alongside M2) |
|
||||||
|
|
||||||
The original first code-bearing step (narrow `ReplicationEventSink` /
|
The original first code-bearing step (narrow `ReplicationEventSink` /
|
||||||
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
||||||
|
|||||||
@@ -1,15 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
pub use super::replication_resync_boundary::ResyncStatusType;
|
|
||||||
@@ -12,7 +12,6 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub mod datatypes;
|
|
||||||
mod replication_bandwidth_boundary;
|
mod replication_bandwidth_boundary;
|
||||||
mod replication_config_boundary;
|
mod replication_config_boundary;
|
||||||
mod replication_config_store;
|
mod replication_config_store;
|
||||||
@@ -29,6 +28,7 @@ mod replication_object_bridge;
|
|||||||
mod replication_object_config;
|
mod replication_object_config;
|
||||||
mod replication_object_decision_boundary;
|
mod replication_object_decision_boundary;
|
||||||
pub(crate) mod replication_pool;
|
pub(crate) mod replication_pool;
|
||||||
|
mod replication_proxy;
|
||||||
mod replication_queue_boundary;
|
mod replication_queue_boundary;
|
||||||
mod replication_resync_boundary;
|
mod replication_resync_boundary;
|
||||||
mod replication_resyncer;
|
mod replication_resyncer;
|
||||||
@@ -43,7 +43,6 @@ pub(crate) mod replication_timing;
|
|||||||
mod replication_versioning_boundary;
|
mod replication_versioning_boundary;
|
||||||
mod runtime_boundary;
|
mod runtime_boundary;
|
||||||
|
|
||||||
pub use datatypes::ResyncStatusType;
|
|
||||||
pub use replication_config_boundary::{
|
pub use replication_config_boundary::{
|
||||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||||
@@ -60,7 +59,7 @@ pub use replication_filemeta_boundary::{
|
|||||||
pub(crate) use replication_filemeta_boundary::{
|
pub(crate) use replication_filemeta_boundary::{
|
||||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||||
};
|
};
|
||||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||||
@@ -74,13 +73,16 @@ pub use replication_pool::{
|
|||||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||||
};
|
};
|
||||||
|
pub use replication_proxy::get_proxy_targets;
|
||||||
pub use replication_queue_boundary::{
|
pub use replication_queue_boundary::{
|
||||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||||
ReplicationPriority, ReplicationQueueAdmission,
|
ReplicationPriority, ReplicationQueueAdmission,
|
||||||
};
|
};
|
||||||
|
pub use replication_resync_boundary::ResyncStatusType;
|
||||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||||
|
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ impl ReplicationConfigStore {
|
|||||||
com::read_config_limited(api, file, max_bytes).await
|
com::read_config_limited(api, file, max_bytes).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||||
where
|
where
|
||||||
S: ReplicationObjectIO,
|
S: ReplicationObjectIO,
|
||||||
|
|||||||
@@ -12,12 +12,11 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) use rustfs_filemeta::NULL_VERSION_ID;
|
|
||||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision,
|
||||||
parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||||
};
|
};
|
||||||
pub use rustfs_replication::{
|
pub use rustfs_replication::{
|
||||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||||
|
|||||||
@@ -24,15 +24,27 @@ use super::replication_storage_boundary::{
|
|||||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||||
|
|
||||||
pub(crate) struct ReplicationLifecycleBridge;
|
pub(crate) struct ReplicationLifecycleBridge;
|
||||||
|
|
||||||
impl ReplicationLifecycleBridge {
|
impl ReplicationLifecycleBridge {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||||
ReplicationConfig::new(Some(config), None)
|
ReplicationConfig::new(Some(config), None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_version_purge(
|
pub(crate) fn has_pending_version_purge(
|
||||||
config: &ReplicationLifecycleConfig,
|
config: &ReplicationLifecycleConfig,
|
||||||
object_name: &str,
|
object_name: &str,
|
||||||
@@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge {
|
|||||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn check_delete_replication(
|
pub(crate) async fn check_delete_replication(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &ObjectToDelete,
|
object: &ObjectToDelete,
|
||||||
@@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge {
|
|||||||
check_replicate_delete(bucket, object, source, opts, None).await
|
check_replicate_delete(bucket, object, source, opts, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||||
let pending_status = decision.pending_status();
|
let pending_status = decision.pending_status();
|
||||||
ReplicationState {
|
ReplicationState {
|
||||||
|
|||||||
@@ -19,17 +19,33 @@ use time::OffsetDateTime;
|
|||||||
use super::replication_error_boundary::Result;
|
use super::replication_error_boundary::Result;
|
||||||
use crate::bucket::msgp_decode;
|
use crate::bucket::msgp_decode;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) struct ReplicationMsgpCodec;
|
pub(crate) struct ReplicationMsgpCodec;
|
||||||
|
|
||||||
impl ReplicationMsgpCodec {
|
impl ReplicationMsgpCodec {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||||
msgp_decode::read_msgp_ext8_time(rd)
|
msgp_decode::read_msgp_ext8_time(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||||
msgp_decode::skip_msgp_value(rd)
|
msgp_decode::skip_msgp_value(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||||
msgp_decode::write_msgp_time(wr, time)
|
msgp_decode::write_msgp_time(wr, time)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,6 +77,10 @@ impl ReplicationObjectBridge {
|
|||||||
load_delete_request_config_in(ctx, bucket).await
|
load_delete_request_config_in(ctx, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn delete_config_snapshot_in(
|
pub(crate) async fn delete_config_snapshot_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config(
|
|||||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn load_delete_replication_config_in(
|
pub(crate) async fn load_delete_replication_config_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -18,9 +18,10 @@ pub use rustfs_replication::{
|
|||||||
should_use_existing_delete_replication_source,
|
should_use_existing_delete_replication_source,
|
||||||
};
|
};
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject,
|
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||||
delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path,
|
delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object,
|
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||||
should_retry_delete_marker_purge,
|
replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
|
||||||
|
should_retry_delete_marker_purge, target_delete_version_id,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||||
let mut tracker = DurableMrfBacklogTracker {
|
let mut tracker = DurableMrfBacklogTracker {
|
||||||
available: true,
|
available: true,
|
||||||
@@ -663,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
|||||||
Err(EcstoreError::PreconditionFailed)
|
Err(EcstoreError::PreconditionFailed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||||
|
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||||
|
/// node is already processing the backlog.
|
||||||
|
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||||
|
let recovery_lock = match storage
|
||||||
|
.new_ns_lock(
|
||||||
|
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||||
|
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(lock) => lock,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to create the MRF recovery leader lock"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match recovery_lock
|
||||||
|
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(guard) => Some(guard),
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
"Another node is already processing the MRF recovery backlog"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads and decodes the on-disk MRF recovery file.
|
||||||
|
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||||
|
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||||
|
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||||
|
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => {
|
||||||
|
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||||
|
available: true,
|
||||||
|
buckets: Vec::new(),
|
||||||
|
});
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to load MRF recovery file"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match decode_mrf_file(&data) {
|
||||||
|
Ok(v) => Some(v),
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||||
|
);
|
||||||
|
quarantine_mrf_file(storage, &data).await;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays one MRF recovery entry by operation kind.
|
||||||
|
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||||
|
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||||
|
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
match entry.op {
|
||||||
|
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||||
|
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||||
|
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||||
|
/// stale force-delete generations are skipped, and plain deletes are
|
||||||
|
/// reconstructed as heal deletes.
|
||||||
|
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
if should_replay_force_delete_intent(entry) {
|
||||||
|
let operation_id = entry.force_delete_id?;
|
||||||
|
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||||
|
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
} else if entry.force_delete_id.is_some() {
|
||||||
|
Some(ReplicationQueueAdmission::Skipped)
|
||||||
|
} else {
|
||||||
|
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||||
|
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: entry.target_arns.clone(),
|
||||||
|
force_delete_generation: entry.force_delete_generation,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||||
|
/// get_object_info here because the delete-marker or version may
|
||||||
|
/// already be absent from the local store — that is expected.
|
||||||
|
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||||
|
let oi = ObjectInfo {
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||||
|
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||||
|
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The MRF entry does not persist the replication decision and the
|
||||||
|
/// source object is gone, so re-derive the decision from the live
|
||||||
|
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||||
|
/// it on the reconstructed delete. Without this the decision string
|
||||||
|
/// is empty and the delete replicates to zero targets — a silent
|
||||||
|
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||||
|
async fn resolve_mrf_delete_replicate_decision(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
versioned: bool,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicateDecision> {
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||||
|
Ok(None) => None,
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||||
|
&entry.bucket,
|
||||||
|
&ObjectToDelete {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
oi,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(dsc) => Some(dsc),
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||||
|
/// replication decision.
|
||||||
|
fn reconstructed_heal_delete_info(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
dsc: &ReplicateDecision,
|
||||||
|
) -> DeletedObjectReplicationInfo {
|
||||||
|
let mut rstate = oi.replication_state();
|
||||||
|
rstate.replicate_decision_str = dsc.to_string();
|
||||||
|
|
||||||
|
let delete_marker_mtime = entry
|
||||||
|
.delete_marker_mtime
|
||||||
|
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||||
|
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker_version_id: entry.delete_marker_version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
delete_marker_mtime,
|
||||||
|
force_delete: entry.force_delete,
|
||||||
|
replication_state: Some(rstate),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
// Legacy entries predate target admission persistence. They cannot
|
||||||
|
// be safely attributed, so retain the old live-config fallback.
|
||||||
|
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a metadata-kind MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF metadata recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||||
|
/// admitted targets, carrying over the entry's retry count.
|
||||||
|
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||||
|
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||||
|
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||||
|
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||||
|
roi
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||||
|
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||||
|
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||||
|
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||||
|
storage: &Arc<S>,
|
||||||
|
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||||
|
entries: &[MrfReplicateEntry],
|
||||||
|
retry_entries: &[MrfReplicateEntry],
|
||||||
|
) -> Vec<MrfReplicateEntry> {
|
||||||
|
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||||
|
Ok(retained) => retained,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||||
|
);
|
||||||
|
match read_mrf_entries(storage.clone()).await {
|
||||||
|
Ok(current) => current,
|
||||||
|
Err(read_error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %read_error,
|
||||||
|
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||||
|
);
|
||||||
|
entries.to_vec()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||||
struct ResyncActiveConflictError {
|
struct ResyncActiveConflictError {
|
||||||
@@ -712,6 +1078,10 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
|||||||
|
|
||||||
// MRF worker lifecycle
|
// MRF worker lifecycle
|
||||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
mrf_stop_tx: Sender<()>,
|
mrf_stop_tx: Sender<()>,
|
||||||
|
|
||||||
// Worker size tracking
|
// Worker size tracking
|
||||||
@@ -940,6 +1310,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Resizes worker priority and counts
|
/// Resizes worker priority and counts
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn resize_worker_priority(
|
pub async fn resize_worker_priority(
|
||||||
&self,
|
&self,
|
||||||
pri: ReplicationPriority,
|
pri: ReplicationPriority,
|
||||||
@@ -1180,6 +1554,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Queues an MRF save operation
|
/// Queues an MRF save operation
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||||
}
|
}
|
||||||
@@ -1205,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let storage = self.storage.clone();
|
let storage = self.storage.clone();
|
||||||
|
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
let recovery_lock = match storage
|
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||||
.new_ns_lock(
|
return;
|
||||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
|
||||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(lock) => lock,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to create the MRF recovery leader lock"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let recovery_guard = match recovery_lock
|
|
||||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(guard) => guard,
|
|
||||||
Err(_) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
"Another node is already processing the MRF recovery backlog"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||||
Ok(d) => d,
|
return;
|
||||||
Err(EcstoreError::ConfigNotFound) => {
|
|
||||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
|
||||||
available: true,
|
|
||||||
buckets: Vec::new(),
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to load MRF recovery file"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let entries = match decode_mrf_file(&data) {
|
|
||||||
Ok(v) => v,
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
|
||||||
);
|
|
||||||
quarantine_mrf_file(&storage, &data).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||||
|
|
||||||
@@ -1278,187 +1597,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let mut retry_entries = Vec::new();
|
let mut retry_entries = Vec::new();
|
||||||
|
|
||||||
for entry in entries.iter() {
|
for entry in entries.iter() {
|
||||||
let admission = match entry.op {
|
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||||
MrfOpKind::Delete => {
|
continue;
|
||||||
if should_replay_force_delete_intent(entry) {
|
|
||||||
let Some(operation_id) = entry.force_delete_id else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let delete = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
force_delete: true,
|
|
||||||
force_delete_id: Some(operation_id),
|
|
||||||
force_delete_target_arns: entry.target_arns.clone(),
|
|
||||||
force_delete_generation: entry.force_delete_generation,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
} else if entry.force_delete_id.is_some() {
|
|
||||||
ReplicationQueueAdmission::Skipped
|
|
||||||
} else {
|
|
||||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
|
||||||
// get_object_info here because the delete-marker or version may
|
|
||||||
// already be absent from the local store — that is expected.
|
|
||||||
//
|
|
||||||
// The MRF entry does not persist the replication decision and the
|
|
||||||
// source object is gone, so re-derive the decision from the live
|
|
||||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
|
||||||
// it on the reconstructed delete. Without this the decision string
|
|
||||||
// is empty and the delete replicates to zero targets — a silent
|
|
||||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
|
||||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
|
||||||
let oi = ObjectInfo {
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let dsc = if entry.target_arns.is_empty() {
|
|
||||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
|
||||||
Ok(None) => continue,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
|
||||||
&entry.bucket,
|
|
||||||
&ObjectToDelete {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&oi,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(dsc) => dsc,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
|
||||||
};
|
|
||||||
let mut rstate = oi.replication_state();
|
|
||||||
rstate.replicate_decision_str = dsc.to_string();
|
|
||||||
|
|
||||||
let delete_marker_mtime = entry
|
|
||||||
.delete_marker_mtime
|
|
||||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
|
||||||
|
|
||||||
let dv = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker_version_id: entry.delete_marker_version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
delete_marker_mtime,
|
|
||||||
force_delete: entry.force_delete,
|
|
||||||
replication_state: Some(rstate),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
// Legacy entries predate target admission persistence. They cannot
|
|
||||||
// be safely attributed, so retain the old live-config fallback.
|
|
||||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Metadata => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF metadata recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if admission == ReplicationQueueAdmission::Missed {
|
if admission == ReplicationQueueAdmission::Missed {
|
||||||
@@ -1468,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||||
Ok(retained) => retained,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
|
||||||
);
|
|
||||||
match read_mrf_entries(storage.clone()).await {
|
|
||||||
Ok(current) => current,
|
|
||||||
Err(read_error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %read_error,
|
|
||||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
|
||||||
);
|
|
||||||
entries.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let retained_count = retained.len();
|
let retained_count = retained.len();
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||||
|
|
||||||
@@ -1651,6 +1769,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling regular replication operations
|
/// Worker function for handling regular replication operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_worker(
|
async fn add_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1664,6 +1786,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling large object replication operations
|
/// Worker function for handling large object replication operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_large_worker(
|
async fn add_large_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1678,6 +1804,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_mrf_worker(
|
async fn add_mrf_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1691,6 +1821,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Delete resync metadata from replication resync state in memory
|
/// Delete resync metadata from replication resync state in memory
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||||
let mut status_map = self.resyncer.status_map.write().await;
|
let mut status_map = self.resyncer.status_map.write().await;
|
||||||
status_map.remove(bucket);
|
status_map.remove(bucket);
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
//! Proxy-target selection for reads of objects not yet replicated locally
|
||||||
|
//! (MinIO `getProxyTargets`, bucket-replication.go).
|
||||||
|
//!
|
||||||
|
//! During the active-active replication lag window a GET/HEAD/Tagging request
|
||||||
|
//! for an object the local site does not have yet may be served by proxying to
|
||||||
|
//! a replication target. This module only *selects* the candidate targets; the
|
||||||
|
//! request-path callers perform the remote calls and response translation.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tracing::debug;
|
||||||
|
|
||||||
|
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||||
|
use super::replication_object_config::get_replication_config;
|
||||||
|
use super::replication_storage_boundary::ObjectOptions;
|
||||||
|
use super::replication_target_boundary::{ReplicationTargetStore, TargetClient};
|
||||||
|
|
||||||
|
/// Returns the replication-target clients eligible to serve a proxied read of
|
||||||
|
/// `bucket/object`, in rule order. Mirrors MinIO's `getProxyTargets`:
|
||||||
|
///
|
||||||
|
/// - the `source-proxy-request` header family was present at all
|
||||||
|
/// (`opts.proxy_request` / `opts.proxy_header_set`, MinIO `ProxyRequest` /
|
||||||
|
/// `ProxyHeaderSet`) -> empty. "true" is the anti-loop marker of an
|
||||||
|
/// already-proxied client read; "false" is what a peer's replication
|
||||||
|
/// worker sends on convergence HEADs so the receiver answers locally —
|
||||||
|
/// proxying that miss back would echo the source object and fake
|
||||||
|
/// convergence, permanently skipping replication;
|
||||||
|
/// - the bucket's versioning is suspended for the object -> empty;
|
||||||
|
/// - no replication configuration / no matching rule -> empty;
|
||||||
|
/// - otherwise every distinct target ARN whose rules match the object,
|
||||||
|
/// resolved through the bucket target system, skipping targets that opted
|
||||||
|
/// out of proxying (`disable_proxy`).
|
||||||
|
pub async fn get_proxy_targets(bucket: &str, object: &str, opts: &ObjectOptions) -> Vec<Arc<TargetClient>> {
|
||||||
|
if opts.proxy_request || opts.proxy_header_set {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
if opts.version_suspended {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let cfg = match get_replication_config(bucket).await {
|
||||||
|
Ok(Some(cfg)) => cfg,
|
||||||
|
Ok(None) => return Vec::new(),
|
||||||
|
Err(err) => {
|
||||||
|
debug!(bucket, object, error = %err, "read proxy: failed to load replication config; not proxying");
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let arns = cfg.filter_target_arns(&ObjectOpts {
|
||||||
|
name: object.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut targets = Vec::with_capacity(arns.len());
|
||||||
|
for arn in arns {
|
||||||
|
let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||||
|
debug!(bucket, object, arn, "read proxy: no client for replication target ARN");
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if client.disable_proxy {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
targets.push(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
targets
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn opts() -> ObjectOptions {
|
||||||
|
ObjectOptions::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Anti-loop: a request that was already proxied by a peer must never be
|
||||||
|
/// proxied onward, regardless of replication configuration.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn proxy_request_yields_no_targets() {
|
||||||
|
let targets = get_proxy_targets(
|
||||||
|
"bucket",
|
||||||
|
"object",
|
||||||
|
&ObjectOptions {
|
||||||
|
proxy_request: true,
|
||||||
|
..opts()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(targets.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// MinIO `ProxyHeaderSet` parity: the header family being present at all
|
||||||
|
/// disables proxying, even with the value "false" — that is what a
|
||||||
|
/// peer's replication worker sends on convergence HEADs.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn proxy_header_set_yields_no_targets() {
|
||||||
|
let targets = get_proxy_targets(
|
||||||
|
"bucket",
|
||||||
|
"object",
|
||||||
|
&ObjectOptions {
|
||||||
|
proxy_header_set: true,
|
||||||
|
proxy_request: false,
|
||||||
|
..opts()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(targets.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Suspended versioning disables proxying (MinIO parity): the local null
|
||||||
|
/// version is authoritative and a remote read could resurrect data.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn version_suspended_yields_no_targets() {
|
||||||
|
let targets = get_proxy_targets(
|
||||||
|
"bucket",
|
||||||
|
"object",
|
||||||
|
&ObjectOptions {
|
||||||
|
version_suspended: true,
|
||||||
|
..opts()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(targets.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A bucket without replication configuration has nothing to proxy to.
|
||||||
|
/// (No metadata system is running in unit tests, so the config lookup
|
||||||
|
/// resolves to "no configuration" — the same empty-result contract.)
|
||||||
|
#[tokio::test]
|
||||||
|
async fn missing_replication_config_yields_no_targets() {
|
||||||
|
let targets = get_proxy_targets("bucket-without-replication", "object", &opts()).await;
|
||||||
|
assert!(targets.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,17 +15,42 @@
|
|||||||
use super::replication_error_boundary::{Error, Result};
|
use super::replication_error_boundary::{Error, Result};
|
||||||
use super::replication_filemeta_boundary::MrfReplicateEntry;
|
use super::replication_filemeta_boundary::MrfReplicateEntry;
|
||||||
|
|
||||||
|
/// Kept test-only: the runtime consumer was the worker HEAD's fake proxy
|
||||||
|
/// counting (removed in backlog#1675 P1-5); the resyncer tests still pin the
|
||||||
|
/// classifier's semantics for the real client read-proxy failure accounting.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_replication::should_count_head_proxy_failure;
|
||||||
pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus};
|
pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus};
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
is_version_id_mismatch, resync_state_accepts_update, sanitize_resync_error_detail, should_auto_resume_resync,
|
is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
|
||||||
should_count_head_proxy_failure,
|
should_auto_resume_resync,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||||
|
|
||||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -340,6 +340,10 @@ impl ReplicationStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Site replication update replica statistics
|
/// Site replication update replica statistics
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn sr_update_replica_stat(&self, size: i64) {
|
fn sr_update_replica_stat(&self, size: i64) {
|
||||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -704,6 +708,12 @@ impl ReplicationStats {
|
|||||||
} else {
|
} else {
|
||||||
BucketReplicationStats::new()
|
BucketReplicationStats::new()
|
||||||
};
|
};
|
||||||
|
// Stamp the serializable failure windows from the live samples: the
|
||||||
|
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
||||||
|
// is what cluster aggregation and the metrics endpoints see.
|
||||||
|
for stat in replication_stats.stats.values_mut() {
|
||||||
|
stat.fail_stats.refresh_windows();
|
||||||
|
}
|
||||||
let uptime = if cache.contains_key(bucket) {
|
let uptime = if cache.contains_key(bucket) {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(SystemTime::UNIX_EPOCH)
|
.duration_since(SystemTime::UNIX_EPOCH)
|
||||||
@@ -1151,6 +1161,31 @@ mod tests {
|
|||||||
assert!(all.contains_key("proxy-only-bucket"));
|
assert!(all.contains_key("proxy-only-bucket"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pins the read-proxy metric contract (backlog#1675 P1-5): the API
|
||||||
|
/// strings the GET/HEAD/Tagging proxy paths record map onto the
|
||||||
|
/// get/head/tagging totals, and only unexpected failures raise the
|
||||||
|
/// failed counters.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_proxy_stats_map_read_proxy_apis_to_totals() {
|
||||||
|
let stats = ReplicationStats::new();
|
||||||
|
stats.inc_proxy("proxy-bucket", "GetObject", false).await;
|
||||||
|
stats.inc_proxy("proxy-bucket", "GetObject", true).await;
|
||||||
|
stats.inc_proxy("proxy-bucket", "HeadObject", false).await;
|
||||||
|
stats.inc_proxy("proxy-bucket", "GetObjectTagging", false).await;
|
||||||
|
stats.inc_proxy("proxy-bucket", "PutObjectTagging", false).await;
|
||||||
|
stats.inc_proxy("proxy-bucket", "DeleteObjectTagging", true).await;
|
||||||
|
|
||||||
|
let metric = stats.get_proxy_stats("proxy-bucket").await;
|
||||||
|
assert_eq!(metric.get_total, 2);
|
||||||
|
assert_eq!(metric.get_failed, 1);
|
||||||
|
assert_eq!(metric.head_total, 1);
|
||||||
|
assert_eq!(metric.head_failed, 0);
|
||||||
|
assert_eq!(metric.get_tag_total, 1);
|
||||||
|
assert_eq!(metric.put_tag_total, 1);
|
||||||
|
assert_eq!(metric.delete_tag_total, 1);
|
||||||
|
assert_eq!(metric.delete_tag_failed, 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_calculate_bucket_replication_stats_merges_resync_metrics() {
|
async fn test_calculate_bucket_replication_stats_merges_resync_metrics() {
|
||||||
let stats = ReplicationStats::new();
|
let stats = ReplicationStats::new();
|
||||||
|
|||||||
@@ -15,7 +15,9 @@
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_replication::FailStats;
|
pub(crate) use rustfs_replication::FailStats;
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
||||||
SRMetricsSummary, XferStats,
|
|
||||||
};
|
};
|
||||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
||||||
|
// can project the internal stats onto the minio-go response shapes through
|
||||||
|
// the storage_api facade chain.
|
||||||
|
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||||
|
|||||||
@@ -27,18 +27,24 @@ use rustfs_utils::http::{
|
|||||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||||
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
||||||
|
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
||||||
|
ssec_replication_transport_header,
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use time::format_description::well_known::Rfc3339;
|
use time::format_description::well_known::Rfc3339;
|
||||||
|
|
||||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::bucket::target::BucketTarget;
|
pub(crate) use crate::bucket::target::BucketTarget;
|
||||||
pub(crate) use crate::bucket::target::BucketTargets;
|
pub(crate) use crate::bucket::target::BucketTargets;
|
||||||
|
pub use rustfs_replication::SsecPassthroughCapability;
|
||||||
|
pub(crate) use rustfs_replication::{
|
||||||
|
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||||
|
};
|
||||||
|
|
||||||
use super::replication_config_store::ReplicationConfigStore;
|
use super::replication_config_store::ReplicationConfigStore;
|
||||||
use super::replication_error_boundary::{Error, Result};
|
use super::replication_error_boundary::{Error, Result};
|
||||||
@@ -63,6 +69,8 @@ static STANDARD_HEADERS: &[&str] = &[
|
|||||||
];
|
];
|
||||||
|
|
||||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||||
|
pub(crate) const ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED: &str = "replication target does not support SSE-C passthrough: the replica would lose its decryption material \
|
||||||
|
(run ?replication-check to re-probe)";
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
enum ReplicationSourceEncryption {
|
enum ReplicationSourceEncryption {
|
||||||
@@ -119,10 +127,38 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
||||||
|
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
||||||
|
let key = key.as_bytes();
|
||||||
|
key.len() == prefix.len() + suffix.len()
|
||||||
|
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
||||||
|
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
[
|
||||||
|
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|suffix| {
|
||||||
|
["x-rustfs-", "x-minio-"]
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||||
|
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||||
|
/// customer-algorithm header and let the crate-owned policy decide.
|
||||||
|
pub(crate) fn ssec_passthrough_evidence_present(head: &HeadObjectOutput) -> bool {
|
||||||
|
rustfs_replication::ssec_passthrough_evidence_present(head.sse_customer_algorithm.as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) struct ReplicationTargetStore;
|
pub(crate) struct ReplicationTargetStore;
|
||||||
|
|
||||||
impl ReplicationTargetStore {
|
impl ReplicationTargetStore {
|
||||||
@@ -142,6 +178,17 @@ impl ReplicationTargetStore {
|
|||||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||||
|
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||||
|
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||||
|
BucketTargetSys::get()
|
||||||
|
.record_ssec_passthrough_capability(arn, capability)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||||
@@ -176,6 +223,11 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if is_legacy_source_replication_timestamp_key(key) {
|
||||||
|
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if is_internal_key(key) || is_standard_header(key) {
|
if is_internal_key(key) || is_standard_header(key) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -259,15 +311,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
|
|
||||||
if !tags.is_empty() {
|
if !tags.is_empty() {
|
||||||
put_options.user_tags = tags;
|
put_options.user_tags = tags;
|
||||||
put_options.internal.tagging_timestamp =
|
|
||||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
|
||||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
|
||||||
} else {
|
|
||||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Load the stored tagging timestamp independently of whether any tags
|
||||||
|
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||||
|
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||||
|
// With no stored key, fall back to mod_time only while tags exist
|
||||||
|
// (MinIO parity); a tagless object without the key was never tagged and
|
||||||
|
// keeps the epoch default (no header).
|
||||||
|
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||||
|
{
|
||||||
|
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||||
|
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||||
|
} else if !put_options.user_tags.is_empty() {
|
||||||
|
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
|
} else {
|
||||||
|
OffsetDateTime::UNIX_EPOCH
|
||||||
|
};
|
||||||
|
|
||||||
let metadata = &*object_info.user_defined;
|
let metadata = &*object_info.user_defined;
|
||||||
|
|
||||||
@@ -283,13 +343,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
put_options.cache_control = cache_control.to_string();
|
put_options.cache_control = cache_control.to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
||||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
if !retain_until_date.is_empty() {
|
||||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||||
|
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||||
|
}
|
||||||
put_options.internal.retention_timestamp =
|
put_options.internal.retention_timestamp =
|
||||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
@@ -694,6 +756,110 @@ mod tests {
|
|||||||
assert!(options.internal.replication_request);
|
assert!(options.internal.replication_request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||||
|
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||||
|
/// still be loaded (and therefore sent) so the replica can order the
|
||||||
|
/// deletion against concurrent tag edits.
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||||
|
let mut metadata = std::collections::HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||||
|
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
user_tags: Arc::new(String::new()),
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
version_id: Some(Uuid::nil()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||||
|
|
||||||
|
assert!(options.user_tags.is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
options.internal.tagging_timestamp,
|
||||||
|
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||||
|
"the stored tagging timestamp must load independently of remaining tags"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A tagless object without the stored key was never tagged: the epoch
|
||||||
|
// default keeps the header unsent.
|
||||||
|
let untagged = ObjectInfo {
|
||||||
|
user_tags: Arc::new(String::new()),
|
||||||
|
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||||
|
version_id: Some(Uuid::nil()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||||
|
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
||||||
|
let legacy_keys = [
|
||||||
|
"x-rustfs-source-replication-tagging-timestamp",
|
||||||
|
"x-rustfs-source-replication-retention-timestamp",
|
||||||
|
"x-rustfs-source-replication-legalhold-timestamp",
|
||||||
|
"x-minio-source-replication-tagging-timestamp",
|
||||||
|
"x-minio-source-replication-retention-timestamp",
|
||||||
|
"x-minio-source-replication-legalhold-timestamp",
|
||||||
|
];
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(
|
||||||
|
legacy_keys
|
||||||
|
.iter()
|
||||||
|
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||||
|
|
||||||
|
for legacy_key in legacy_keys {
|
||||||
|
assert!(!options.user_metadata.contains_key(legacy_key));
|
||||||
|
assert_eq!(
|
||||||
|
options
|
||||||
|
.user_metadata
|
||||||
|
.get(&format!("x-amz-meta-{legacy_key}"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("2099-01-02T03:04:05Z")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
||||||
|
let mut metadata = HashMap::from([
|
||||||
|
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
||||||
|
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
||||||
|
]);
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
||||||
|
|
||||||
|
assert!(options.mode.is_none());
|
||||||
|
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(
|
||||||
|
options.internal.retention_timestamp,
|
||||||
|
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
||||||
|
);
|
||||||
|
let headers = options.header();
|
||||||
|
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
||||||
|
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
||||||
|
Some("2026-01-02T03:04:05Z")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||||
@@ -756,6 +922,27 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pins the HeadObjectOutput field extraction feeding the crate-owned
|
||||||
|
/// evidence judgment (the gate/evidence policy matrix itself is pinned in
|
||||||
|
/// `rustfs-replication`'s object tests).
|
||||||
|
#[test]
|
||||||
|
fn ssec_passthrough_evidence_requires_customer_algorithm_echo() {
|
||||||
|
let with_evidence = HeadObjectOutput::builder().sse_customer_algorithm("AES256").build();
|
||||||
|
assert!(ssec_passthrough_evidence_present(&with_evidence));
|
||||||
|
|
||||||
|
let empty_algorithm = HeadObjectOutput::builder().sse_customer_algorithm("").build();
|
||||||
|
assert!(
|
||||||
|
!ssec_passthrough_evidence_present(&empty_algorithm),
|
||||||
|
"an empty echo is not evidence of preserved SSE-C material"
|
||||||
|
);
|
||||||
|
|
||||||
|
let without_evidence = HeadObjectOutput::builder().e_tag("\"abc\"").content_length(8).build();
|
||||||
|
assert!(
|
||||||
|
!ssec_passthrough_evidence_present(&without_evidence),
|
||||||
|
"a plain HEAD response must classify the target as having dropped the material"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replication_put_options_adds_ssec_checksum_metadata() {
|
fn replication_put_options_adds_ssec_checksum_metadata() {
|
||||||
let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||||
|
|||||||
@@ -40,7 +40,14 @@ impl ARN {
|
|||||||
|
|
||||||
impl Display for ARN {
|
impl Display for ARN {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
// The `minio` partition is deliberate: madmin-go's ParseARN
|
||||||
|
// hard-rejects any other partition, so native mc/madmin tooling can
|
||||||
|
// only decode remote-target ARNs minted in this form (backlog#1675
|
||||||
|
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
||||||
|
// readable via the FromStr whitelist below; runtime matching between
|
||||||
|
// targets and replication rules is by full-string equality, so mixed
|
||||||
|
// partitions coexist safely.
|
||||||
|
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,7 +55,12 @@ impl FromStr for ARN {
|
|||||||
type Err = std::io::Error;
|
type Err = std::io::Error;
|
||||||
|
|
||||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
if !s.starts_with("arn:rustfs:") {
|
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
||||||
|
// from_str(...).unwrap_or_default()` below never fails, so this is
|
||||||
|
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
||||||
|
// the legacy partition and must stay accepted forever (persisted
|
||||||
|
// bucket-targets.json / replication configs from older releases).
|
||||||
|
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,14 +113,50 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// RustFS commonly generates ARNs with an empty region:
|
/// RustFS commonly generates ARNs with an empty region:
|
||||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
||||||
#[test]
|
#[test]
|
||||||
fn from_str_handles_empty_region_segment() {
|
fn from_str_handles_empty_region_segment() {
|
||||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||||
|
|
||||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||||
assert_eq!(parsed.id, "depl-123");
|
assert_eq!(parsed.id, "depl-123");
|
||||||
assert_eq!(parsed.bucket, "bucket-a");
|
assert_eq!(parsed.bucket, "bucket-a");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
||||||
|
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
||||||
|
/// native mc/madmin tooling cannot decode remote-target listings.
|
||||||
|
#[test]
|
||||||
|
fn display_emits_minio_partition() {
|
||||||
|
let arn = ARN::new(
|
||||||
|
BucketTargetType::ReplicationService,
|
||||||
|
"depl-123".to_string(),
|
||||||
|
String::new(),
|
||||||
|
"bucket-a".to_string(),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted bucket-targets.json files from older RustFS releases carry
|
||||||
|
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
||||||
|
#[test]
|
||||||
|
fn from_str_accepts_legacy_rustfs_partition() {
|
||||||
|
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
||||||
|
|
||||||
|
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||||
|
assert_eq!(parsed.region, "us-east-1");
|
||||||
|
assert_eq!(parsed.id, "depl-123");
|
||||||
|
assert_eq!(parsed.bucket, "bucket-a");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
||||||
|
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
||||||
|
/// would otherwise parse as `type=None`.
|
||||||
|
#[test]
|
||||||
|
fn from_str_rejects_unknown_partition() {
|
||||||
|
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
||||||
|
assert!(ARN::from_str("not-an-arn").is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ impl fmt::Debug for Credentials {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub enum ServiceType {
|
pub enum ServiceType {
|
||||||
#[default]
|
#[default]
|
||||||
Replication,
|
Replication,
|
||||||
|
|||||||
@@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
|||||||
check_bucket_name_common(bucket_name, true)
|
check_bucket_name_common(bucket_name, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
|
||||||
if object_name.len() > 1024 {
|
|
||||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
|
||||||
}
|
|
||||||
if !object_name.is_ascii() {
|
|
||||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
|
||||||
if object_name.trim().is_empty() {
|
|
||||||
return Err(Error::other("Object name cannot be empty"));
|
|
||||||
}
|
|
||||||
check_valid_object_name_prefix(object_name)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||||
where
|
where
|
||||||
T: for<'xml> xml::Deserialize<'xml>,
|
T: for<'xml> xml::Deserialize<'xml>,
|
||||||
@@ -100,6 +83,10 @@ where
|
|||||||
Ok(ans)
|
Ok(ans)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||||
let mut buf = Vec::with_capacity(256);
|
let mut buf = Vec::with_capacity(256);
|
||||||
{
|
{
|
||||||
@@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
|||||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||||
/// accept the full S3 key character set.
|
/// accept the full S3 key character set.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||||
|
|
||||||
/// Client-facing reason for path segments Windows can store but not address
|
/// Client-facing reason for path segments Windows can store but not address
|
||||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||||
|
|
||||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||||
@@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
|||||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||||
object.split(['/', '\\']).any(|segment| {
|
object.split(['/', '\\']).any(|segment| {
|
||||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||||
|
|||||||
@@ -90,6 +90,10 @@ impl BucketVersioningSys {
|
|||||||
/// caller's own instance context so a second in-process store never
|
/// caller's own instance context so a second in-process store never
|
||||||
/// answers with the first instance's versioning state; falls back to the
|
/// answers with the first instance's versioning state; falls back to the
|
||||||
/// ambient system when the instance cell is not initialized.
|
/// ambient system when the instance cell is not initialized.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||||
return Ok(VersioningConfiguration::default());
|
return Ok(VersioningConfiguration::default());
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||||
use crate::disk::error::DiskError;
|
use crate::disk::error::DiskError;
|
||||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||||
|
use futures::future::join_all;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||||
use std::{
|
use std::{
|
||||||
@@ -655,6 +656,7 @@ async fn list_path_raw_inner(
|
|||||||
errs.push(None);
|
errs.push(None);
|
||||||
}
|
}
|
||||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||||
|
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut current = MetaCacheEntry::default();
|
let mut current = MetaCacheEntry::default();
|
||||||
@@ -676,6 +678,21 @@ async fn list_path_raw_inner(
|
|||||||
let mut has_err = 0;
|
let mut has_err = 0;
|
||||||
let mut agree = 0;
|
let mut agree = 0;
|
||||||
|
|
||||||
|
// Start every missing head read in the same round so one stalled
|
||||||
|
// disk cannot multiply the wait budget by the erasure-set width.
|
||||||
|
// Outcomes are still consumed below in stable disk-index order.
|
||||||
|
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||||
|
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cancel = &revjob_rx;
|
||||||
|
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
||||||
|
});
|
||||||
|
for (i, outcome) in join_all(concurrent_peeks).await {
|
||||||
|
peek_outcomes[i] = Some(outcome);
|
||||||
|
}
|
||||||
|
|
||||||
for (i, r) in readers.iter_mut().enumerate() {
|
for (i, r) in readers.iter_mut().enumerate() {
|
||||||
if errs[i].is_some() {
|
if errs[i].is_some() {
|
||||||
has_err += 1;
|
has_err += 1;
|
||||||
@@ -685,7 +702,10 @@ async fn list_path_raw_inner(
|
|||||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||||
entry
|
entry
|
||||||
} else {
|
} else {
|
||||||
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
let Some(outcome) = peek_outcomes[i].take() else {
|
||||||
|
return Err(DiskError::Unexpected);
|
||||||
|
};
|
||||||
|
match outcome {
|
||||||
PeekOutcome::Ready(res) => {
|
PeekOutcome::Ready(res) => {
|
||||||
if let Some(entry) = res {
|
if let Some(entry) = res {
|
||||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||||
@@ -1295,6 +1315,36 @@ mod tests {
|
|||||||
assert_eq!(err, DiskError::Timeout);
|
assert_eq!(err, DiskError::Timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
||||||
|
let peek_timeout = Duration::from_millis(20);
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let err = list_path_raw(
|
||||||
|
CancellationToken::new(),
|
||||||
|
ListPathRawOptions {
|
||||||
|
disks: vec![None, None, None, None],
|
||||||
|
min_disks: 1,
|
||||||
|
test_reader_behaviors: vec![
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
],
|
||||||
|
peek_timeout: Some(peek_timeout),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("all stalled readers should fail the listing");
|
||||||
|
|
||||||
|
assert_eq!(err, DiskError::Timeout);
|
||||||
|
assert_eq!(
|
||||||
|
started.elapsed(),
|
||||||
|
peek_timeout,
|
||||||
|
"reader deadlines must overlap instead of accumulating once per disk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||||
let entry = MetaCacheEntry {
|
let entry = MetaCacheEntry {
|
||||||
|
|||||||
@@ -229,17 +229,6 @@ pub fn http_resp_to_error_response(
|
|||||||
err_resp
|
err_resp
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
|
||||||
ErrorResponse {
|
|
||||||
status_code: StatusCode::BAD_REQUEST,
|
|
||||||
code: S3ErrorCode::InvalidArgument,
|
|
||||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
|
||||||
.to_string(),
|
|
||||||
bucket_name: bucket_name.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||||
let msg = format!(
|
let msg = format!(
|
||||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||||
@@ -295,16 +284,6 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
|
||||||
ErrorResponse {
|
|
||||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
|
||||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
|
||||||
message: message.to_string(),
|
|
||||||
request_id: "rustfs".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -95,7 +95,6 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct GetRequest {
|
pub struct GetRequest {
|
||||||
pub buffer: Vec<u8>,
|
pub buffer: Vec<u8>,
|
||||||
pub offset: i64,
|
pub offset: i64,
|
||||||
@@ -107,11 +106,12 @@ pub struct GetRequest {
|
|||||||
pub setting_object_info: bool,
|
pub setting_object_info: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct GetResponse {
|
pub struct GetResponse {
|
||||||
pub size: i64,
|
pub size: i64,
|
||||||
//pub error: error,
|
//pub error: error,
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
pub did_read: bool,
|
pub did_read: bool,
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
pub object_info: ObjectInfo,
|
pub object_info: ObjectInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,6 +135,10 @@ impl Object {
|
|||||||
Self { ..Default::default() }
|
Self { ..Default::default() }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||||
let _ = request.did_offset_change;
|
let _ = request.did_offset_change;
|
||||||
let _ = request.offset;
|
let _ = request.offset;
|
||||||
@@ -150,12 +154,20 @@ impl Object {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||||
self.curr_offset += bytes_read;
|
self.curr_offset += bytes_read;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||||
let mut read_req = GetRequest {
|
let mut read_req = GetRequest {
|
||||||
is_read_op: true,
|
is_read_op: true,
|
||||||
@@ -180,6 +192,10 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let _ = self.do_get_request(&GetRequest {
|
let _ = self.do_get_request(&GetRequest {
|
||||||
@@ -192,6 +208,10 @@ impl Object {
|
|||||||
Ok(self.object_info.clone())
|
Ok(self.object_info.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||||
self.curr_offset = offset;
|
self.curr_offset = offset;
|
||||||
|
|
||||||
@@ -219,6 +239,10 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let seek_req = GetRequest {
|
let seek_req = GetRequest {
|
||||||
@@ -253,6 +277,10 @@ impl Object {
|
|||||||
Ok(self.curr_offset)
|
Ok(self.curr_offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||||
self.is_closed = true;
|
self.is_closed = true;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||||
|
use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
@@ -27,7 +28,6 @@ use tracing::warn;
|
|||||||
use crate::client::api_error_response::err_invalid_argument;
|
use crate::client::api_error_response::err_invalid_argument;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct AdvancedGetOptions {
|
pub struct AdvancedGetOptions {
|
||||||
pub replication_delete_marker: bool,
|
pub replication_delete_marker: bool,
|
||||||
pub is_replication_ready_for_delete_marker: bool,
|
pub is_replication_ready_for_delete_marker: bool,
|
||||||
@@ -77,7 +77,7 @@ impl GetObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if self.checksum {
|
if self.checksum {
|
||||||
headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
|
headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
|
||||||
}
|
}
|
||||||
headers
|
headers
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -360,7 +360,6 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct ListObjectsOptions {
|
pub struct ListObjectsOptions {
|
||||||
reverse_versions: bool,
|
reverse_versions: bool,
|
||||||
with_versions: bool,
|
with_versions: bool,
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ use crate::client::{
|
|||||||
api_put_object_common::optimal_part_info,
|
api_put_object_common::optimal_part_info,
|
||||||
api_put_object_multipart::UploadPartParams,
|
api_put_object_multipart::UploadPartParams,
|
||||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
||||||
credentials::SignatureType,
|
credentials::SignatureType,
|
||||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl PutObjectOptions {
|
impl PutObjectOptions {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn set_match_etag(&mut self, etag: &str) {
|
fn set_match_etag(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||||
@@ -149,6 +149,7 @@ impl PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn set_match_etag_except(&mut self, etag: &str) {
|
fn set_match_etag_except(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||||
@@ -259,6 +260,7 @@ impl PutObjectOptions {
|
|||||||
header
|
header
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||||
//if self.checksum.is_set() {
|
//if self.checksum.is_set() {
|
||||||
/*if !self.trailing_header_support {
|
/*if !self.trailing_header_support {
|
||||||
|
|||||||
@@ -30,10 +30,6 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
|||||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
|
||||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||||
let unknown_size;
|
let unknown_size;
|
||||||
let mut object_size = object_size;
|
let mut object_size = object_size;
|
||||||
|
|||||||
@@ -81,18 +81,6 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct UploadedPartRes {
|
|
||||||
pub error: std::io::Error,
|
|
||||||
pub part_num: i64,
|
|
||||||
pub size: i64,
|
|
||||||
pub part: ObjectPart,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct UploadPartReq {
|
|
||||||
pub part_num: i64,
|
|
||||||
pub part: ObjectPart,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TransitionClient {
|
impl TransitionClient {
|
||||||
pub async fn put_object_multipart_stream(
|
pub async fn put_object_multipart_stream(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
|
|||||||
@@ -55,7 +55,6 @@ pub struct RemoveBucketOptions {
|
|||||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct AdvancedRemoveOptions {
|
pub struct AdvancedRemoveOptions {
|
||||||
pub replication_delete_marker: bool,
|
pub replication_delete_marker: bool,
|
||||||
pub replication_status: ReplicationStatus,
|
pub replication_status: ReplicationStatus,
|
||||||
@@ -465,10 +464,10 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct RemoveObjectError {
|
pub struct RemoveObjectError {
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
object_name: String,
|
object_name: String,
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
version_id: String,
|
version_id: String,
|
||||||
err: Option<std::io::Error>,
|
err: Option<std::io::Error>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,10 +29,6 @@ use crate::client::utils::base64_decode;
|
|||||||
|
|
||||||
use super::transition_api;
|
use super::transition_api;
|
||||||
|
|
||||||
pub struct ListAllMyBucketsResult {
|
|
||||||
pub owner: Owner,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
pub struct CommonPrefix {
|
pub struct CommonPrefix {
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
@@ -89,6 +85,10 @@ pub struct ListVersionsResult {
|
|||||||
pub next_version_id_marker: String,
|
pub next_version_id_marker: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct ListBucketResult {
|
pub struct ListBucketResult {
|
||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
contents: Vec<transition_api::ObjectInfo>,
|
contents: Vec<transition_api::ObjectInfo>,
|
||||||
@@ -102,6 +102,10 @@ pub struct ListBucketResult {
|
|||||||
prefix: String,
|
prefix: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct ListMultipartUploadsResult {
|
pub struct ListMultipartUploadsResult {
|
||||||
bucket: String,
|
bucket: String,
|
||||||
key_marker: String,
|
key_marker: String,
|
||||||
@@ -117,16 +121,15 @@ pub struct ListMultipartUploadsResult {
|
|||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct Initiator {
|
pub struct Initiator {
|
||||||
id: String,
|
id: String,
|
||||||
display_name: String,
|
display_name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CopyObjectResult {
|
|
||||||
pub etag: String,
|
|
||||||
pub last_modified: OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ObjectPart {
|
pub struct ObjectPart {
|
||||||
pub etag: String,
|
pub etag: String,
|
||||||
@@ -260,6 +263,7 @@ pub struct CompletePart {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CompletePart {
|
impl CompletePart {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||||
match t {
|
match t {
|
||||||
ChecksumMode::ChecksumCRC32C => {
|
ChecksumMode::ChecksumCRC32C => {
|
||||||
@@ -284,11 +288,6 @@ impl CompletePart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CopyObjectPartResult {
|
|
||||||
pub etag: String,
|
|
||||||
pub last_modified: OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Default, serde::Serialize)]
|
#[derive(Debug, Default, serde::Serialize)]
|
||||||
#[serde(rename = "CompleteMultipartUpload")]
|
#[serde(rename = "CompleteMultipartUpload")]
|
||||||
pub struct CompleteMultipartUpload {
|
pub struct CompleteMultipartUpload {
|
||||||
@@ -357,10 +356,10 @@ impl CompleteMultipartUpload {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CreateBucketConfiguration {
|
#[allow(
|
||||||
pub location: String,
|
dead_code,
|
||||||
}
|
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
||||||
|
)]
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteObject {
|
pub struct DeleteObject {
|
||||||
//api has
|
//api has
|
||||||
@@ -368,21 +367,6 @@ pub struct DeleteObject {
|
|||||||
pub version_id: String,
|
pub version_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct DeletedObject {
|
|
||||||
//s3s has
|
|
||||||
pub key: String,
|
|
||||||
pub version_id: String,
|
|
||||||
pub deletemarker: bool,
|
|
||||||
pub deletemarker_version_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct NonDeletedObject {
|
|
||||||
pub key: String,
|
|
||||||
pub code: String,
|
|
||||||
pub message: String,
|
|
||||||
pub version_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteMultiObjects {
|
pub struct DeleteMultiObjects {
|
||||||
pub quiet: bool,
|
pub quiet: bool,
|
||||||
@@ -402,6 +386,7 @@ impl DeleteMultiObjects {
|
|||||||
Ok(buf)
|
Ok(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
||||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct WireDeleteObject {
|
struct WireDeleteObject {
|
||||||
@@ -436,8 +421,3 @@ impl DeleteMultiObjects {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct DeleteMultiObjectsResult {
|
|
||||||
pub deleted_objects: Vec<DeletedObject>,
|
|
||||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -365,11 +365,15 @@ mod tests {
|
|||||||
pub struct Checksum {
|
pub struct Checksum {
|
||||||
checksum_type: ChecksumMode,
|
checksum_type: ChecksumMode,
|
||||||
r: Vec<u8>,
|
r: Vec<u8>,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
||||||
|
)]
|
||||||
computed: bool,
|
computed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl Checksum {
|
impl Checksum {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||||
return Checksum {
|
return Checksum {
|
||||||
@@ -381,7 +385,7 @@ impl Checksum {
|
|||||||
Checksum::default()
|
Checksum::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||||
let b = match base64_decode(s.as_bytes()) {
|
let b = match base64_decode(s.as_bytes()) {
|
||||||
Ok(b) => b,
|
Ok(b) => b,
|
||||||
@@ -408,7 +412,7 @@ impl Checksum {
|
|||||||
base64_encode(&self.r)
|
base64_encode(&self.r)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn raw(&self) -> Option<Vec<u8>> {
|
fn raw(&self) -> Option<Vec<u8>> {
|
||||||
if !self.is_set() {
|
if !self.is_set() {
|
||||||
return None;
|
return None;
|
||||||
|
|||||||
@@ -32,8 +32,5 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
|||||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||||
|
|
||||||
pub const TOTAL_WORKERS: i64 = 4;
|
|
||||||
|
|
||||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
|
||||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||||
|
|||||||
@@ -67,6 +67,10 @@ impl<P: Provider + Default> Credentials<P> {
|
|||||||
Ok(self.creds.clone())
|
Ok(self.creds.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn expire(&mut self) {
|
fn expire(&mut self) {
|
||||||
self.force_refresh = true;
|
self.force_refresh = true;
|
||||||
}
|
}
|
||||||
@@ -133,6 +137,10 @@ impl Provider for Static {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct STSError {
|
pub struct STSError {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub r#type: String,
|
pub r#type: String,
|
||||||
pub code: String,
|
pub code: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
@@ -141,6 +149,10 @@ pub struct STSError {
|
|||||||
#[derive(Debug, Clone, thiserror::Error)]
|
#[derive(Debug, Clone, thiserror::Error)]
|
||||||
pub struct ErrorResponse {
|
pub struct ErrorResponse {
|
||||||
pub sts_error: STSError,
|
pub sts_error: STSError,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub request_id: String,
|
pub request_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,22 +170,3 @@ impl ErrorResponse {
|
|||||||
return self.sts_error.message.clone();
|
return self.sts_error.message.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
|
||||||
where
|
|
||||||
for<'de> T: Deserialize<'de>,
|
|
||||||
{
|
|
||||||
match std::str::from_utf8(body) {
|
|
||||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
|
||||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
|
||||||
where
|
|
||||||
for<'de> T: Deserialize<'de>,
|
|
||||||
{
|
|
||||||
let body = body_reader.to_vec();
|
|
||||||
let parsed = xml_decoder(&body)?;
|
|
||||||
Ok((body, parsed))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod admin_handler_utils;
|
pub mod admin_handler_utils;
|
||||||
pub mod api_error_response;
|
pub mod api_error_response;
|
||||||
|
|||||||
@@ -37,16 +37,17 @@ pub struct PutObjReader {
|
|||||||
//pub sealMD5Fn: SealMD5CurrFn,
|
//pub sealMD5Fn: SealMD5CurrFn,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl PutObjReader {
|
impl PutObjReader {
|
||||||
pub fn new(reader: HashReader) -> Self {
|
pub fn new(reader: HashReader) -> Self {
|
||||||
Self { reader }
|
Self { reader }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn md5_current_hex_string(&self) -> String {
|
fn md5_current_hex_string(&self) -> String {
|
||||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||||
self.reader = enc_reader;
|
self.reader = enc_reader;
|
||||||
|
|
||||||
@@ -77,39 +78,6 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
|
||||||
let mut skip_length: i64 = 0;
|
|
||||||
let mut cumulative_actual_size: i64 = 0;
|
|
||||||
let mut first_part_idx: i64 = 0;
|
|
||||||
let mut compressed_offset: i64 = 0;
|
|
||||||
let mut part_skip: i64 = 0;
|
|
||||||
let mut decrypt_skip: i64 = 0;
|
|
||||||
let mut seq_num: u64 = 0;
|
|
||||||
for (i, part) in oi.parts.iter().enumerate() {
|
|
||||||
cumulative_actual_size += part.actual_size as i64;
|
|
||||||
if cumulative_actual_size <= offset {
|
|
||||||
compressed_offset += part.size as i64;
|
|
||||||
} else {
|
|
||||||
first_part_idx = i as i64;
|
|
||||||
skip_length = cumulative_actual_size - part.actual_size as i64;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
skip_length = offset - skip_length;
|
|
||||||
|
|
||||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
|
||||||
if skip_length > 0
|
|
||||||
&& parts.len() > first_part_idx as usize
|
|
||||||
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
|
||||||
{
|
|
||||||
let _ = part_skip;
|
|
||||||
let _ = decrypt_skip;
|
|
||||||
let _ = seq_num;
|
|
||||||
}
|
|
||||||
|
|
||||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_getobjectreader<'a>(
|
pub fn new_getobjectreader<'a>(
|
||||||
rs: &Option<HTTPRangeSpec>,
|
rs: &Option<HTTPRangeSpec>,
|
||||||
oi: &'a ObjectInfo,
|
oi: &'a ObjectInfo,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
|
|||||||
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||||
pub(crate) enum BucketVersioningState {
|
pub(crate) enum BucketVersioningState {
|
||||||
Unknown,
|
Unknown,
|
||||||
Disabled,
|
Disabled,
|
||||||
@@ -47,6 +48,7 @@ impl RemoteVersion {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||||
match self {
|
match self {
|
||||||
Self::Unknown => Err(Error::new(
|
Self::Unknown => Err(Error::new(
|
||||||
|
|||||||
@@ -54,6 +54,10 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
|||||||
use rustfs_rio::HashReader;
|
use rustfs_rio::HashReader;
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::{
|
use rustfs_utils::{
|
||||||
|
http::headers::{
|
||||||
|
AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1,
|
||||||
|
AMZ_CHECKSUM_SHA256,
|
||||||
|
},
|
||||||
net::get_endpoint_url,
|
net::get_endpoint_url,
|
||||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||||
};
|
};
|
||||||
@@ -101,6 +105,10 @@ where
|
|||||||
|
|
||||||
const C_UNKNOWN: i32 = -1;
|
const C_UNKNOWN: i32 = -1;
|
||||||
const C_OFFLINE: i32 = 0;
|
const C_OFFLINE: i32 = 0;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reachable only from the unused transition client methods below (backlog#1823)"
|
||||||
|
)]
|
||||||
const C_ONLINE: i32 = 1;
|
const C_ONLINE: i32 = 1;
|
||||||
|
|
||||||
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
||||||
@@ -320,6 +328,10 @@ impl TransitionClient {
|
|||||||
Ok(client)
|
Ok(client)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn endpoint_url(&self) -> Url {
|
fn endpoint_url(&self) -> Url {
|
||||||
self.endpoint_url.clone()
|
self.endpoint_url.clone()
|
||||||
}
|
}
|
||||||
@@ -348,12 +360,20 @@ impl TransitionClient {
|
|||||||
.to_string())
|
.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn trace_errors_only_off(&self) {
|
fn trace_errors_only_off(&self) {
|
||||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||||
*trace_errors_only = false;
|
*trace_errors_only = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn trace_off(&self) {
|
fn trace_off(&self) {
|
||||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||||
*is_trace_enabled = false;
|
*is_trace_enabled = false;
|
||||||
@@ -363,12 +383,20 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||||
*endpoint = accelerate_endpoint.to_string();
|
*endpoint = accelerate_endpoint.to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||||
*dual_stack = enabled;
|
*dual_stack = enabled;
|
||||||
@@ -398,10 +426,18 @@ impl TransitionClient {
|
|||||||
(hash_algos, hash_sums)
|
(hash_algos, hash_sums)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn is_online(&self) -> bool {
|
fn is_online(&self) -> bool {
|
||||||
!self.is_offline()
|
!self.is_offline()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn mark_offline(&self) {
|
fn mark_offline(&self) {
|
||||||
self.health_status
|
self.health_status
|
||||||
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
||||||
@@ -411,10 +447,18 @@ impl TransitionClient {
|
|||||||
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn health_check(hc_duration: Duration) {
|
fn health_check(hc_duration: Duration) {
|
||||||
let _ = hc_duration;
|
let _ = hc_duration;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
||||||
let mut resp_trace: Vec<u8>;
|
let mut resp_trace: Vec<u8>;
|
||||||
|
|
||||||
@@ -1102,6 +1146,7 @@ impl Default for ObjectInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectInfo {
|
impl ObjectInfo {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn remote_version(
|
pub(crate) fn remote_version(
|
||||||
&self,
|
&self,
|
||||||
capabilities: ProviderVersionCapabilities,
|
capabilities: ProviderVersionCapabilities,
|
||||||
@@ -1342,12 +1387,12 @@ pub(crate) fn to_object_info_for_provider(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Extract checksums
|
// Extract checksums
|
||||||
let checksum_crc32 = get_header("x-amz-checksum-crc32");
|
let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32);
|
||||||
let checksum_crc32c = get_header("x-amz-checksum-crc32c");
|
let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C);
|
||||||
let checksum_sha1 = get_header("x-amz-checksum-sha1");
|
let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1);
|
||||||
let checksum_sha256 = get_header("x-amz-checksum-sha256");
|
let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256);
|
||||||
let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
|
let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME);
|
||||||
let checksum_mode = get_header("x-amz-checksum-mode");
|
let checksum_mode = get_header(AMZ_CHECKSUM_MODE);
|
||||||
|
|
||||||
// Build and return the ObjectInfo struct
|
// Build and return the ObjectInfo struct
|
||||||
Ok(ObjectInfo {
|
Ok(ObjectInfo {
|
||||||
|
|||||||
@@ -48,10 +48,6 @@ lazy_static! {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
|
||||||
SUPPORTED_QUERY_VALUES[qs_key]
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_storageclass_header(header_key: &str) -> bool {
|
pub fn is_storageclass_header(header_key: &str) -> bool {
|
||||||
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,6 +86,25 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
|||||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Error for a peer that reported `success = false` without an `error_info` payload.
|
||||||
|
///
|
||||||
|
/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError`
|
||||||
|
/// instead of `DiskError`. The message names the operation (and the bucket, where the
|
||||||
|
/// operation has one) and nothing else, for two reasons:
|
||||||
|
///
|
||||||
|
/// - `finalize_result` classifies failures by message substring, so any text matching
|
||||||
|
/// `message_has_network_needle` would take an answering peer offline and evict its
|
||||||
|
/// connection over a plain application-level rejection.
|
||||||
|
/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered
|
||||||
|
/// message, so a per-peer detail such as the peer address would split one shared
|
||||||
|
/// failure into single-count buckets and downgrade the dominant error.
|
||||||
|
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||||
|
match bucket {
|
||||||
|
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||||
|
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
||||||
if !response.success {
|
if !response.success {
|
||||||
return Err(Error::other(
|
return Err(Error::other(
|
||||||
@@ -696,7 +715,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("local_storage_info", None));
|
||||||
}
|
}
|
||||||
let data = response.storage_info;
|
let data = response.storage_info;
|
||||||
|
|
||||||
@@ -719,7 +738,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("server_info", None));
|
||||||
}
|
}
|
||||||
let data = response.server_properties;
|
let data = response.server_properties;
|
||||||
|
|
||||||
@@ -742,7 +761,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_cpus", None));
|
||||||
}
|
}
|
||||||
let data = response.cpus;
|
let data = response.cpus;
|
||||||
|
|
||||||
@@ -765,7 +784,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_net_info", None));
|
||||||
}
|
}
|
||||||
let data = response.net_info;
|
let data = response.net_info;
|
||||||
|
|
||||||
@@ -788,7 +807,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_partitions", None));
|
||||||
}
|
}
|
||||||
let data = response.partitions;
|
let data = response.partitions;
|
||||||
|
|
||||||
@@ -811,7 +830,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_os_info", None));
|
||||||
}
|
}
|
||||||
let data = response.os_info;
|
let data = response.os_info;
|
||||||
|
|
||||||
@@ -832,7 +851,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_se_linux_info", None));
|
||||||
}
|
}
|
||||||
let data = response.sys_services;
|
let data = response.sys_services;
|
||||||
|
|
||||||
@@ -857,7 +876,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_sys_config", None));
|
||||||
}
|
}
|
||||||
let data = response.sys_config;
|
let data = response.sys_config;
|
||||||
|
|
||||||
@@ -882,7 +901,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_sys_errors", None));
|
||||||
}
|
}
|
||||||
let data = response.sys_errors;
|
let data = response.sys_errors;
|
||||||
|
|
||||||
@@ -907,7 +926,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_mem_info", None));
|
||||||
}
|
}
|
||||||
let data = response.mem_info;
|
let data = response.mem_info;
|
||||||
|
|
||||||
@@ -939,7 +958,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_metrics", None));
|
||||||
}
|
}
|
||||||
let data = response.realtime_metrics;
|
let data = response.realtime_metrics;
|
||||||
|
|
||||||
@@ -964,7 +983,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_live_events", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(PeerLiveEventsBatch {
|
Ok(PeerLiveEventsBatch {
|
||||||
@@ -989,7 +1008,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("get_proc_info", None));
|
||||||
}
|
}
|
||||||
let data = response.proc_info;
|
let data = response.proc_info;
|
||||||
|
|
||||||
@@ -1016,7 +1035,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("start_profiling", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1323,7 +1342,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1346,7 +1365,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket)));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1369,7 +1388,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("delete_policy", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1392,7 +1411,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_policy", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1417,7 +1436,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_policy_mapping", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1440,7 +1459,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("delete_user", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1463,7 +1482,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("delete_service_account", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1487,7 +1506,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_user", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1510,7 +1529,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_service_account", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1533,7 +1552,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_group", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1554,7 +1573,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("reload_site_replication_config", None));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1597,7 +1616,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("signal_service", None));
|
||||||
}
|
}
|
||||||
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
||||||
Ok(response)
|
Ok(response)
|
||||||
@@ -1667,7 +1686,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("reload_pool_meta", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1691,7 +1710,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("stop_rebalance", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1725,7 +1744,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("load_rebalance_meta", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1753,7 +1772,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("start_decommission", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1777,7 +1796,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("decommission_cancel", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1801,7 +1820,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(Error::other(""));
|
return Err(peer_failure_without_details("clear_decommission", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1947,6 +1966,8 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||||
|
use crate::disk::error::DiskError;
|
||||||
|
use crate::disk::error_reduce::reduce_errs;
|
||||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -3098,4 +3119,115 @@ mod tests {
|
|||||||
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every operation name passed to `peer_failure_without_details` in this file.
|
||||||
|
const PEER_FAILURE_OPS: &[&str] = &[
|
||||||
|
"local_storage_info",
|
||||||
|
"server_info",
|
||||||
|
"get_cpus",
|
||||||
|
"get_net_info",
|
||||||
|
"get_partitions",
|
||||||
|
"get_os_info",
|
||||||
|
"get_se_linux_info",
|
||||||
|
"get_sys_config",
|
||||||
|
"get_sys_errors",
|
||||||
|
"get_mem_info",
|
||||||
|
"get_metrics",
|
||||||
|
"get_live_events",
|
||||||
|
"get_proc_info",
|
||||||
|
"start_profiling",
|
||||||
|
"load_bucket_metadata",
|
||||||
|
"delete_bucket_metadata",
|
||||||
|
"delete_policy",
|
||||||
|
"load_policy",
|
||||||
|
"load_policy_mapping",
|
||||||
|
"delete_user",
|
||||||
|
"delete_service_account",
|
||||||
|
"load_user",
|
||||||
|
"load_service_account",
|
||||||
|
"load_group",
|
||||||
|
"reload_site_replication_config",
|
||||||
|
"signal_service",
|
||||||
|
"reload_pool_meta",
|
||||||
|
"stop_rebalance",
|
||||||
|
"load_rebalance_meta",
|
||||||
|
"start_decommission",
|
||||||
|
"decommission_cancel",
|
||||||
|
"clear_decommission",
|
||||||
|
];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||||
|
for op in PEER_FAILURE_OPS {
|
||||||
|
let message = peer_failure_without_details(op, None).to_string();
|
||||||
|
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
for op in ["load_bucket_metadata", "delete_bucket_metadata"] {
|
||||||
|
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||||
|
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||||
|
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||||
|
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||||
|
// same operation must stay a single dominant error instead of one bucket per peer.
|
||||||
|
let per_peer_errs = (0..4)
|
||||||
|
.map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let (count, dominant) = reduce_errs(&per_peer_errs, &[]);
|
||||||
|
assert_eq!(count, 4, "one shared failure must not split into per-peer buckets");
|
||||||
|
assert_eq!(
|
||||||
|
dominant,
|
||||||
|
Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_ne!(
|
||||||
|
peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(),
|
||||||
|
peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string()
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(),
|
||||||
|
peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_failure_without_details_never_reads_as_a_network_failure() {
|
||||||
|
// `finalize_result` marks the peer offline and evicts its connection whenever the
|
||||||
|
// message matches a network needle. A peer that answered `success = false` is alive,
|
||||||
|
// so no operation or bucket name may push this text over that classifier.
|
||||||
|
for op in PEER_FAILURE_OPS {
|
||||||
|
let err = peer_failure_without_details(op, None);
|
||||||
|
assert!(
|
||||||
|
!PeerRestClient::is_network_like_error(&err),
|
||||||
|
"{op} must not read as a transport failure: {err}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let scoped = peer_failure_without_details(op, Some("bucket-name"));
|
||||||
|
assert!(
|
||||||
|
!PeerRestClient::is_network_like_error(&scoped),
|
||||||
|
"{op} must not read as a transport failure: {scoped}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bucket name is caller-supplied. Every needle carries a space, which S3 bucket
|
||||||
|
// names cannot, and the name is closed by `)` before the literal text resumes, so no
|
||||||
|
// needle can straddle the boundary either.
|
||||||
|
for bucket in [
|
||||||
|
"timed-out",
|
||||||
|
"connection-reset",
|
||||||
|
"transport-error",
|
||||||
|
"broken-pipe",
|
||||||
|
"unavailable-logs",
|
||||||
|
] {
|
||||||
|
let err = peer_failure_without_details("load_bucket_metadata", Some(bucket));
|
||||||
|
assert!(
|
||||||
|
!PeerRestClient::is_network_like_error(&err),
|
||||||
|
"bucket {bucket} must not push the message over the network classifier: {err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -214,6 +214,21 @@ fn pool_write_quorum(participant_count: usize) -> usize {
|
|||||||
(participant_count / 2) + 1
|
(participant_count / 2) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Error for a peer that reported `success = false` without an error payload.
|
||||||
|
///
|
||||||
|
/// The message must stay identical across the peers of one operation: `reduce_errs`
|
||||||
|
/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address,
|
||||||
|
/// timing) would split one shared failure into single-count buckets and downgrade a real
|
||||||
|
/// dominant error into `ErasureWriteQuorum`.
|
||||||
|
///
|
||||||
|
/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape.
|
||||||
|
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||||
|
match bucket {
|
||||||
|
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||||
|
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
||||||
if per_pool_errs.is_empty() {
|
if per_pool_errs.is_empty() {
|
||||||
return Some(Error::ErasureWriteQuorum);
|
return Some(Error::ErasureWriteQuorum);
|
||||||
@@ -1078,7 +1093,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::other(""))
|
Err(peer_failure_without_details("heal_bucket", Some(bucket)))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1105,7 +1120,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::other(""))
|
Err(peer_failure_without_details("list_bucket", None))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let bucket_infos = response
|
let bucket_infos = response
|
||||||
@@ -1136,9 +1151,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::other(format!(
|
Err(peer_failure_without_details("make_bucket", Some(bucket)))
|
||||||
"make_bucket({bucket}): peer returned failure without error details"
|
|
||||||
)))
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1162,7 +1175,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::other(""))
|
Err(peer_failure_without_details("get_bucket_info", Some(bucket)))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
||||||
@@ -1190,7 +1203,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(Error::other(""))
|
Err(peer_failure_without_details("delete_bucket", Some(bucket)))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2314,4 +2327,37 @@ mod tests {
|
|||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||||
|
for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] {
|
||||||
|
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||||
|
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||||
|
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let message = peer_failure_without_details("list_bucket", None).to_string();
|
||||||
|
assert!(message.contains("list_bucket"), "cluster-wide message must name the operation");
|
||||||
|
assert!(!message.trim().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||||
|
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||||
|
// same operation on the same bucket must still reach quorum as one dominant error.
|
||||||
|
let per_pool_errs = vec![
|
||||||
|
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||||
|
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||||
|
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||||
|
];
|
||||||
|
assert_eq!(
|
||||||
|
reduce_pool_write_quorum_errs(&per_pool_errs),
|
||||||
|
Some(peer_failure_without_details("delete_bucket", Some("shared")))
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_ne!(
|
||||||
|
peer_failure_without_details("delete_bucket", Some("shared")),
|
||||||
|
peer_failure_without_details("get_bucket_info", Some("shared"))
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,6 @@ use rustfs_config::{
|
|||||||
};
|
};
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[allow(clippy::declare_interior_mutable_const)]
|
#[allow(clippy::declare_interior_mutable_const)]
|
||||||
/// Default KVS for audit webhook settings.
|
/// Default KVS for audit webhook settings.
|
||||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
@@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
])
|
])
|
||||||
});
|
});
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[allow(clippy::declare_interior_mutable_const)]
|
#[allow(clippy::declare_interior_mutable_const)]
|
||||||
/// Default KVS for audit MQTT settings.
|
/// Default KVS for audit MQTT settings.
|
||||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
@@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
])
|
])
|
||||||
});
|
});
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
KVS(vec![
|
KVS(vec![
|
||||||
KV {
|
KV {
|
||||||
|
|||||||
@@ -12,12 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::error::{Error, Result};
|
|
||||||
use rustfs_config::server_config::{KV, KVS};
|
use rustfs_config::server_config::{KV, KVS};
|
||||||
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
||||||
use rustfs_utils::string::parse_bool;
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
KVS(vec![KV {
|
KVS(vec![KV {
|
||||||
@@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
hidden_if_empty: false,
|
hidden_if_empty: false,
|
||||||
}])
|
}])
|
||||||
});
|
});
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
pub struct Config {
|
|
||||||
pub bitrot: String,
|
|
||||||
pub sleep: Duration,
|
|
||||||
pub io_count: usize,
|
|
||||||
pub drive_workers: usize,
|
|
||||||
pub cache: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Config {
|
|
||||||
pub fn bitrot_scan_cycle(&self) -> Duration {
|
|
||||||
self.cache
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_workers(&self) -> usize {
|
|
||||||
self.drive_workers
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn update(&mut self, nopts: &Config) {
|
|
||||||
self.bitrot = nopts.bitrot.clone();
|
|
||||||
self.io_count = nopts.io_count;
|
|
||||||
self.sleep = nopts.sleep;
|
|
||||||
self.drive_workers = nopts.drive_workers;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
|
||||||
|
|
||||||
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
|
||||||
match parse_bool(s) {
|
|
||||||
Ok(enabled) => {
|
|
||||||
if enabled {
|
|
||||||
Ok(Duration::from_secs_f64(0.0))
|
|
||||||
} else {
|
|
||||||
Ok(Duration::from_secs_f64(-1.0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
if !s.ends_with("m") {
|
|
||||||
return Err(Error::other("unknown format"));
|
|
||||||
}
|
|
||||||
|
|
||||||
match s.trim_end_matches('m').parse::<u64>() {
|
|
||||||
Ok(months) => {
|
|
||||||
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
|
||||||
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
|
||||||
}
|
|
||||||
Err(err) => Err(Error::other(err)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,7 +16,6 @@
|
|||||||
|
|
||||||
mod audit;
|
mod audit;
|
||||||
pub mod com;
|
pub mod com;
|
||||||
#[allow(dead_code)]
|
|
||||||
pub mod heal;
|
pub mod heal;
|
||||||
mod notify;
|
mod notify;
|
||||||
mod oidc;
|
mod oidc;
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta;
|
|||||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||||
use crate::bucket::{
|
use crate::bucket::{
|
||||||
lifecycle::{
|
lifecycle::{
|
||||||
|
LifecycleExpiryConfigs,
|
||||||
bucket_lifecycle_audit::LcEventSrc,
|
bucket_lifecycle_audit::LcEventSrc,
|
||||||
bucket_lifecycle_ops::{
|
bucket_lifecycle_ops::{
|
||||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||||
@@ -1996,11 +1997,11 @@ impl PoolMeta {
|
|||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||||
struct PoolInfo {
|
struct PoolInfo {
|
||||||
position: usize,
|
position: usize,
|
||||||
completed: bool,
|
completed: bool,
|
||||||
|
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||||
decom_started: bool,
|
decom_started: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
|
||||||
|
action.delete()
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
||||||
if !apply_actions || applied {
|
if !apply_actions || applied {
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
@@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct HealLifecycleExpiryContext {
|
||||||
|
configs: LifecycleExpiryConfigs,
|
||||||
|
}
|
||||||
|
|
||||||
impl ECStore {
|
impl ECStore {
|
||||||
|
pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||||
|
if bucket == RUSTFS_META_BUCKET {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let configs = get_expiry_configs(self, bucket).await?;
|
||||||
|
if configs.lifecycle.is_none() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Some(HealLifecycleExpiryContext { configs }))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn enqueue_heal_lifecycle_expiry(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
context: &HealLifecycleExpiryContext,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
version_id: Option<&str>,
|
||||||
|
object_info: Option<&crate::object_api::ObjectInfo>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
let object_info = if let Some(object_info) = object_info {
|
||||||
|
if object_info.bucket != bucket || object_info.name != object {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let snapshot_version_id = object_info
|
||||||
|
.version_id
|
||||||
|
.filter(|version_id| !version_id.is_nil())
|
||||||
|
.map(|version_id| version_id.to_string());
|
||||||
|
if snapshot_version_id.as_deref() != version_id {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
object_info.clone()
|
||||||
|
} else {
|
||||||
|
match self
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
version_id: version_id.map(str::to_string),
|
||||||
|
versioned: version_id.is_some(),
|
||||||
|
expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(object_info) => object_info,
|
||||||
|
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
|
||||||
|
if !lifecycle_action_skips_heal_version(event.action) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
|
||||||
|
}
|
||||||
|
|
||||||
async fn save_current_pool_meta(&self) -> Result<()> {
|
async fn save_current_pool_meta(&self) -> Result<()> {
|
||||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||||
let snapshot = {
|
let snapshot = {
|
||||||
@@ -4287,6 +4365,19 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lifecycle_action_skips_heal_version_for_every_delete_action() {
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
|
||||||
|
assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
|
||||||
|
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
|
||||||
|
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
|
||||||
|
assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
||||||
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
||||||
@@ -4958,13 +5049,19 @@ fn is_disk_online_state(state: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||||
|
)]
|
||||||
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||||
fallback_total_capacity_dedup(disks)
|
fallback_total_capacity_dedup(disks)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||||
|
)]
|
||||||
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||||
fallback_free_capacity_dedup(disks)
|
fallback_free_capacity_dedup(disks)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
|||||||
|
|
||||||
Err(Error::DiskNotFound)
|
Err(Error::DiskNotFound)
|
||||||
}
|
}
|
||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||||
// Multipart orphan reconciliation is intentionally retained above the pool/set layers
|
self.get_disks_for_heal_object(object, opts)?
|
||||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
.check_abandoned_parts(bucket, object, opts)
|
||||||
Err(StorageError::NotImplemented)
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1996,7 +1996,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() {
|
async fn sets_check_abandoned_parts_rejects_invalid_set_scope() {
|
||||||
let format = FormatV3::new(1, 1);
|
let format = FormatV3::new(1, 1);
|
||||||
let sets = Sets {
|
let sets = Sets {
|
||||||
id: format.id,
|
id: format.id,
|
||||||
@@ -2021,10 +2021,21 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let err = sets
|
let err = sets
|
||||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
.check_abandoned_parts(
|
||||||
|
"bucket",
|
||||||
|
"object",
|
||||||
|
&HealOpts {
|
||||||
|
set: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.expect_err("abandoned-parts ownership should stay above the pool/set storage layers");
|
.expect_err("out-of-range abandoned-parts set scope must fail closed");
|
||||||
assert!(matches!(err, StorageError::NotImplemented));
|
assert!(
|
||||||
|
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||||
|
if field == "set" && reason.contains("invalid heal set index 1")),
|
||||||
|
"unexpected invalid set error: {err:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
||||||
|
|||||||
@@ -190,6 +190,17 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
|
|||||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
||||||
@@ -551,6 +562,32 @@ mod tests {
|
|||||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||||
|
"data_read_inline_body_verify"
|
||||||
|
);
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||||
|
"data_read_inline_identity_mismatch"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||||
|
"data_read_inline_missing_payload"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
|
||||||
|
"data_read_inline_missing_shard"
|
||||||
|
);
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||||
|
"data_read_inline_transformed"
|
||||||
|
);
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
||||||
|
|||||||
@@ -637,14 +637,23 @@ impl Default for DiskOperationMetrics {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl DiskOperationMetrics {
|
impl DiskOperationMetrics {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_call(&mut self) {
|
fn record_call(&mut self) {
|
||||||
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_latency_atomic(now_sec, elapsed);
|
self.record_latency_atomic(now_sec, elapsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
|
||||||
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_call();
|
self.record_call();
|
||||||
self.record_latency(now_sec, elapsed);
|
self.record_latency(now_sec, elapsed);
|
||||||
@@ -770,6 +779,7 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Set disk as faulty
|
/// Set disk as faulty
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn set_faulty(&self) {
|
pub fn set_faulty(&self) {
|
||||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||||
}
|
}
|
||||||
@@ -850,6 +860,7 @@ impl DiskHealthTracker {
|
|||||||
became_offline
|
became_offline
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||||
let current = self.runtime_state();
|
let current = self.runtime_state();
|
||||||
if current == RuntimeDriveHealthState::Offline {
|
if current == RuntimeDriveHealthState::Offline {
|
||||||
@@ -980,11 +991,13 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get waiting operations count
|
/// Get waiting operations count
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn waiting_count(&self) -> u32 {
|
pub fn waiting_count(&self) -> u32 {
|
||||||
self.waiting.load(Ordering::Relaxed)
|
self.waiting.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get last success timestamp
|
/// Get last success timestamp
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn last_success(&self) -> i64 {
|
pub fn last_success(&self) -> i64 {
|
||||||
self.last_success.load(Ordering::Acquire)
|
self.last_success.load(Ordering::Acquire)
|
||||||
}
|
}
|
||||||
@@ -1026,21 +1039,6 @@ impl Default for DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Health check context key for tracking disk operations
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
struct HealthDiskCtxKey;
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct HealthDiskCtxValue {
|
|
||||||
last_success: Arc<AtomicI64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl HealthDiskCtxValue {
|
|
||||||
fn log_success(&self) {
|
|
||||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -1072,10 +1070,6 @@ impl LocalDiskWrapper {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
|
|
||||||
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn new_with_reconnect_state(
|
pub(crate) fn new_with_reconnect_state(
|
||||||
disk: Arc<LocalDisk>,
|
disk: Arc<LocalDisk>,
|
||||||
health_check: bool,
|
health_check: bool,
|
||||||
@@ -1438,20 +1432,6 @@ impl LocalDiskWrapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
|
|
||||||
if want_id.is_none() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
let stored_disk_id = self.disk.get_disk_id().await?;
|
|
||||||
|
|
||||||
if stored_disk_id != want_id {
|
|
||||||
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Check if disk ID is stale
|
/// Check if disk ID is stale
|
||||||
async fn check_disk_stale(&self) -> Result<()> {
|
async fn check_disk_stale(&self) -> Result<()> {
|
||||||
let Some(current_disk_id) = *self.disk_id.read().await else {
|
let Some(current_disk_id) = *self.disk_id.read().await else {
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
||||||
match io_err.kind() {
|
match io_err.kind() {
|
||||||
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||||
match fs::remove_file(path.as_ref()).await {
|
match fs::remove_file(path.as_ref()).await {
|
||||||
|
|||||||
@@ -665,6 +665,7 @@ async fn remove_empty_directory_tree_under_mount_lease(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn remove_empty_directory_tree_with(
|
async fn remove_empty_directory_tree_with(
|
||||||
root: &Path,
|
root: &Path,
|
||||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||||
@@ -1016,13 +1017,29 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
|
|||||||
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
||||||
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
||||||
/// Default: false (uses page cache via mmap/pread).
|
/// Default: false (uses page cache via mmap/pread).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
||||||
|
|
||||||
/// Minimum shard size threshold for O_DIRECT reads.
|
/// Minimum shard size threshold for O_DIRECT reads.
|
||||||
/// Only shards larger than this threshold will use O_DIRECT.
|
/// Only shards larger than this threshold will use O_DIRECT.
|
||||||
/// Default: 4MB.
|
/// Default: 4MB.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
||||||
@@ -1036,7 +1053,15 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
|||||||
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
||||||
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
||||||
/// Default: false (buffered writes via the page cache, as before).
|
/// Default: false (buffered writes via the page cache, as before).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||||
@@ -1095,12 +1120,14 @@ macro_rules! cached_read_env {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT reads are enabled.
|
/// Check if O_DIRECT reads are enabled.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn is_direct_io_read_enabled() -> bool =
|
fn is_direct_io_read_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT shard/part data writes are enabled.
|
/// Check if O_DIRECT shard/part data writes are enabled.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn is_direct_io_write_enabled() -> bool =
|
fn is_direct_io_write_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
||||||
}
|
}
|
||||||
@@ -1456,6 +1483,7 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Get the O_DIRECT read threshold size.
|
/// Get the O_DIRECT read threshold size.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn get_direct_io_read_threshold() -> usize =
|
fn get_direct_io_read_threshold() -> usize =
|
||||||
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
||||||
}
|
}
|
||||||
@@ -1673,12 +1701,20 @@ impl DirectIoWriteState {
|
|||||||
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
||||||
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
||||||
/// into one positioned write to keep the syscall count low.
|
/// into one positioned write to keep the syscall count low.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
||||||
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
||||||
/// flushed batch length, and every write offset stay alignment-correct.
|
/// flushed batch length, and every write offset stay alignment-correct.
|
||||||
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
fn direct_write_staging_capacity(align: usize) -> usize {
|
fn direct_write_staging_capacity(align: usize) -> usize {
|
||||||
debug_assert!(align.is_power_of_two() && align >= 512);
|
debug_assert!(align.is_power_of_two() && align >= 512);
|
||||||
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
||||||
@@ -1687,6 +1723,10 @@ fn direct_write_staging_capacity(align: usize) -> usize {
|
|||||||
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
||||||
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
||||||
/// so the tail-boundary math is unit-tested on any host.
|
/// so the tail-boundary math is unit-tested on any host.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
||||||
let aligned = filled - (filled % align);
|
let aligned = filled - (filled % align);
|
||||||
(aligned, filled - aligned)
|
(aligned, filled - aligned)
|
||||||
@@ -2142,6 +2182,7 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
||||||
DELETE_VERSION_FAIL_AFTER_COMMIT
|
DELETE_VERSION_FAIL_AFTER_COMMIT
|
||||||
.lock()
|
.lock()
|
||||||
@@ -2447,6 +2488,10 @@ enum SyncMode {
|
|||||||
FileOnly,
|
FileOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||||
|
)]
|
||||||
struct FileCacheReclaimWriter {
|
struct FileCacheReclaimWriter {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_len: usize,
|
reclaim_len: usize,
|
||||||
@@ -2454,6 +2499,10 @@ struct FileCacheReclaimWriter {
|
|||||||
reclaimed: bool,
|
reclaimed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||||
|
)]
|
||||||
struct FileCacheReclaimReader {
|
struct FileCacheReclaimReader {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_offset: u64,
|
reclaim_offset: u64,
|
||||||
@@ -2519,6 +2568,10 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||||
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
||||||
// (three metric-key constructions) when general metrics are disabled.
|
// (three metric-key constructions) when general metrics are disabled.
|
||||||
@@ -3071,6 +3124,7 @@ impl LocalIoBackend for StdBackend {
|
|||||||
use memmap2::MmapOptions;
|
use memmap2::MmapOptions;
|
||||||
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
|
||||||
struct MmapCopyReadResult {
|
struct MmapCopyReadResult {
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
access_check_duration: StdDuration,
|
access_check_duration: StdDuration,
|
||||||
@@ -4704,6 +4758,10 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
|||||||
Arc::new(StdBackend::new(root))
|
Arc::new(StdBackend::new(root))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct LocalDisk {
|
pub struct LocalDisk {
|
||||||
pub root: PathBuf,
|
pub root: PathBuf,
|
||||||
publication_root: os::PublicationRoot,
|
publication_root: os::PublicationRoot,
|
||||||
@@ -5490,6 +5548,7 @@ impl LocalDisk {
|
|||||||
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
||||||
let path_ref = path.as_ref();
|
let path_ref = path.as_ref();
|
||||||
let path_str = path_ref.to_string_lossy();
|
let path_str = path_ref.to_string_lossy();
|
||||||
@@ -5567,15 +5626,24 @@ impl LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check if a path is valid
|
// Check if a path is valid
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||||
check_local_disk_valid_path(self.io_root(), path)
|
check_local_disk_valid_path(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
||||||
reject_local_disk_symlink_components(self.io_root(), path)
|
reject_local_disk_symlink_components(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch path generation with single lock acquisition
|
// Batch path generation with single lock acquisition
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
||||||
let mut results = Vec::with_capacity(requests.len());
|
let mut results = Vec::with_capacity(requests.len());
|
||||||
let mut cache_misses = Vec::new();
|
let mut cache_misses = Vec::new();
|
||||||
@@ -6488,12 +6556,13 @@ impl LocalDisk {
|
|||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn get_metrics(&self) -> DiskMetrics {
|
fn get_metrics(&self) -> DiskMetrics {
|
||||||
DiskMetrics::default()
|
DiskMetrics::default()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod disk_store;
|
pub mod disk_store;
|
||||||
pub mod endpoint;
|
pub mod endpoint;
|
||||||
@@ -1114,6 +1113,10 @@ pub struct DiskInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct Info {
|
pub struct Info {
|
||||||
pub total: u64,
|
pub total: u64,
|
||||||
pub free: u64,
|
pub free: u64,
|
||||||
@@ -1372,6 +1375,7 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
||||||
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -571,6 +571,10 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
|||||||
|
|
||||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||||
/// itself.
|
/// itself.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
@@ -583,6 +587,7 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
|
|
||||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||||
/// concurrently, bounded both per directory and process-wide.
|
/// concurrently, bounded both per directory and process-wide.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
||||||
}
|
}
|
||||||
@@ -1809,10 +1814,6 @@ impl RenameCommitGuard {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
|
|
||||||
self.destination_directory_guard(directory, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn create_destination_directory_for_path_access(
|
pub(crate) fn create_destination_directory_for_path_access(
|
||||||
&self,
|
&self,
|
||||||
directory: &Path,
|
directory: &Path,
|
||||||
@@ -2858,13 +2859,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a file exists.
|
|
||||||
/// Returns true if the file exists, false otherwise.
|
|
||||||
#[tracing::instrument(level = "debug", skip_all)]
|
|
||||||
pub fn file_exists(path: impl AsRef<Path>) -> bool {
|
|
||||||
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether an [`io::Error`] means "the directory is not empty".
|
/// Whether an [`io::Error`] means "the directory is not empty".
|
||||||
///
|
///
|
||||||
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
||||||
|
|||||||
@@ -820,10 +820,263 @@ impl BitrotWriterWrapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
||||||
|
//
|
||||||
|
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
||||||
|
// key-handling regression) fails silently: every shard reads back "corrupt",
|
||||||
|
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
||||||
|
// which copy is healthy. The self-test below pins the algorithms the moment a
|
||||||
|
// process starts, so a drifted build announces itself instead of quietly
|
||||||
|
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
||||||
|
// analysis-2026-08-16.md §6 HS-11.
|
||||||
|
|
||||||
|
/// Length of the deterministic self-test payload.
|
||||||
|
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
||||||
|
|
||||||
|
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
||||||
|
/// (the production default). Pinned so any platform or build where the
|
||||||
|
/// implementation drifts fails startup instead of miss-hashing shards.
|
||||||
|
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
||||||
|
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
||||||
|
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
||||||
|
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
||||||
|
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
||||||
|
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
||||||
|
];
|
||||||
|
|
||||||
|
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
||||||
|
/// Highway digests above this one is externally verifiable, so it guards the
|
||||||
|
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
||||||
|
/// self-computed constants.
|
||||||
|
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
||||||
|
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
||||||
|
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
||||||
|
/// platform and every run hashes the same 4096 bytes.
|
||||||
|
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
||||||
|
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||||
|
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
||||||
|
for byte in payload.iter_mut() {
|
||||||
|
state ^= state >> 12;
|
||||||
|
state ^= state << 25;
|
||||||
|
state ^= state >> 27;
|
||||||
|
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
||||||
|
}
|
||||||
|
payload
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a bitrot self-test failed.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum BitrotSelfTestError {
|
||||||
|
/// A known-answer digest mismatched the pinned constant.
|
||||||
|
KnownAnswerMismatch {
|
||||||
|
algorithm: &'static str,
|
||||||
|
got: String,
|
||||||
|
want: String,
|
||||||
|
},
|
||||||
|
/// A freshly encoded shard failed `bitrot_verify`.
|
||||||
|
RoundtripVerify { algorithm: &'static str, detail: String },
|
||||||
|
/// A verified roundtrip read back different bytes than were written.
|
||||||
|
RoundtripReadback { algorithm: &'static str },
|
||||||
|
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
||||||
|
TamperNotRejected {
|
||||||
|
algorithm: &'static str,
|
||||||
|
tampered: &'static str,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for BitrotSelfTestError {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
||||||
|
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
||||||
|
}
|
||||||
|
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
||||||
|
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
||||||
|
Self::TamperNotRejected { algorithm, tampered } => {
|
||||||
|
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for BitrotSelfTestError {}
|
||||||
|
|
||||||
|
fn self_test_hex(bytes: &[u8]) -> String {
|
||||||
|
rustfs_utils::hex(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// (kept as a named one-liner so every KAT failure site reads the same; the
|
||||||
|
// underlying formatter is the shared `rustfs_utils::hex`)
|
||||||
|
|
||||||
|
/// Compare a digest against its pinned constant. Split out so a test can drive
|
||||||
|
/// it with a wrong constant and prove the mismatch path fires.
|
||||||
|
fn bitrot_kat_check(
|
||||||
|
algorithm: &'static str,
|
||||||
|
algo: &HashAlgorithm,
|
||||||
|
payload: &[u8],
|
||||||
|
expected: &[u8; 32],
|
||||||
|
) -> Result<(), BitrotSelfTestError> {
|
||||||
|
let digest = algo.hash_encode(payload);
|
||||||
|
let digest = digest.as_ref();
|
||||||
|
if digest.len() != expected.len() || digest != expected.as_slice() {
|
||||||
|
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||||
|
algorithm,
|
||||||
|
got: self_test_hex(digest),
|
||||||
|
want: self_test_hex(expected),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
||||||
|
/// every block back through `BitrotReader` comparing bytes.
|
||||||
|
async fn bitrot_roundtrip_check(
|
||||||
|
algorithm: &'static str,
|
||||||
|
algo: HashAlgorithm,
|
||||||
|
payload: &[u8],
|
||||||
|
shard_size: usize,
|
||||||
|
) -> Result<(), BitrotSelfTestError> {
|
||||||
|
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||||
|
for chunk in payload.chunks(shard_size) {
|
||||||
|
writer
|
||||||
|
.write(chunk)
|
||||||
|
.await
|
||||||
|
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||||
|
algorithm,
|
||||||
|
detail: format!("encode failed: {err}"),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
let encoded = writer.into_inner().into_inner();
|
||||||
|
|
||||||
|
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||||
|
if encoded.len() != on_disk {
|
||||||
|
return Err(BitrotSelfTestError::RoundtripVerify {
|
||||||
|
algorithm,
|
||||||
|
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
||||||
|
.await
|
||||||
|
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||||
|
algorithm,
|
||||||
|
detail: err.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
||||||
|
let mut offset = 0usize;
|
||||||
|
while offset < payload.len() {
|
||||||
|
let want = shard_size.min(payload.len() - offset);
|
||||||
|
let mut buf = vec![0u8; want];
|
||||||
|
let read = reader
|
||||||
|
.read(&mut buf)
|
||||||
|
.await
|
||||||
|
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||||
|
algorithm,
|
||||||
|
detail: format!("read back failed at offset {offset}: {err}"),
|
||||||
|
})?;
|
||||||
|
if read != want || buf[..read] != payload[offset..offset + read] {
|
||||||
|
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
||||||
|
}
|
||||||
|
offset += read;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flip one byte and require `bitrot_verify` to reject the result.
|
||||||
|
async fn bitrot_tamper_check(
|
||||||
|
algorithm: &'static str,
|
||||||
|
algo: HashAlgorithm,
|
||||||
|
payload: &[u8],
|
||||||
|
shard_size: usize,
|
||||||
|
tampered: &'static str,
|
||||||
|
flip_at: usize,
|
||||||
|
) -> Result<(), BitrotSelfTestError> {
|
||||||
|
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||||
|
for chunk in payload.chunks(shard_size) {
|
||||||
|
writer.write(chunk).await.expect("self-test encode should not fail");
|
||||||
|
}
|
||||||
|
let mut corrupt = writer.into_inner().into_inner();
|
||||||
|
let flip_index = flip_at % corrupt.len();
|
||||||
|
corrupt[flip_index] ^= 0x80;
|
||||||
|
|
||||||
|
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||||
|
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
||||||
|
// The flipped byte must be rejected as a hash mismatch specifically, not
|
||||||
|
// by any incidental read error: an in-memory cursor cannot fail reads,
|
||||||
|
// so accepting any other failure here would mask a verify path that
|
||||||
|
// errors out before it ever compares hashes.
|
||||||
|
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
||||||
|
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
||||||
|
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
||||||
|
algorithm,
|
||||||
|
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify every bitrot algorithm this crate can write or verify in production:
|
||||||
|
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
||||||
|
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
||||||
|
/// and the leading hash, while all three hashed algorithms reproduce their
|
||||||
|
/// pinned known-answer digests.
|
||||||
|
///
|
||||||
|
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
||||||
|
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
||||||
|
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
||||||
|
let payload = bitrot_self_test_payload();
|
||||||
|
|
||||||
|
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
||||||
|
// itself, before any self-pinned constants are consulted.
|
||||||
|
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
||||||
|
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
||||||
|
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||||
|
algorithm: "SHA256",
|
||||||
|
got: self_test_hex(abc.as_ref()),
|
||||||
|
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
bitrot_kat_check(
|
||||||
|
"HighwayHash256S",
|
||||||
|
&HashAlgorithm::HighwayHash256S,
|
||||||
|
&payload,
|
||||||
|
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
||||||
|
)?;
|
||||||
|
bitrot_kat_check(
|
||||||
|
"HighwayHash256SLegacy",
|
||||||
|
&HashAlgorithm::HighwayHash256SLegacy,
|
||||||
|
&payload,
|
||||||
|
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
for (algorithm, algo) in [
|
||||||
|
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
||||||
|
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
||||||
|
] {
|
||||||
|
// Full blocks plus a partial tail, exactly like a real part stripe.
|
||||||
|
let tail_len = 2 * 1024 + 333;
|
||||||
|
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
||||||
|
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
||||||
|
// One flipped byte in the final data block, one in the first leading
|
||||||
|
// hash: both must fail verification.
|
||||||
|
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
||||||
|
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
||||||
|
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||||
};
|
};
|
||||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -1090,6 +1343,32 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bitrot_self_test_payload_is_deterministic() {
|
||||||
|
// Two independent builds of the payload must agree byte for byte, or
|
||||||
|
// the pinned known-answer digests below would be meaningless.
|
||||||
|
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
||||||
|
let payload = bitrot_self_test_payload();
|
||||||
|
let wrong = [0u8; 32];
|
||||||
|
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
||||||
|
.expect_err("a zeroed digest must never match");
|
||||||
|
match err {
|
||||||
|
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
||||||
|
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bitrot_self_test_passes() {
|
||||||
|
bitrot_self_test()
|
||||||
|
.await
|
||||||
|
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||||
let mut counting = VectoredCountingWriter::default();
|
let mut counting = VectoredCountingWriter::default();
|
||||||
@@ -1189,7 +1468,7 @@ mod tests {
|
|||||||
let last = corrupt.len() - 1;
|
let last = corrupt.len() - 1;
|
||||||
corrupt[last] ^= 0x80;
|
corrupt[last] ^= 0x80;
|
||||||
let err = bitrot_verify(
|
let err = bitrot_verify(
|
||||||
Cursor::new(corrupt),
|
std::io::Cursor::new(corrupt),
|
||||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||||
data.len(),
|
data.len(),
|
||||||
algo,
|
algo,
|
||||||
@@ -1282,7 +1561,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||||
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||||
let mut out = [0u8; 5];
|
let mut out = [0u8; 5];
|
||||||
let err = reader
|
let err = reader
|
||||||
.read(&mut out)
|
.read(&mut out)
|
||||||
@@ -1407,7 +1686,7 @@ mod tests {
|
|||||||
(HashAlgorithm::HighwayHash256, true),
|
(HashAlgorithm::HighwayHash256, true),
|
||||||
] {
|
] {
|
||||||
let label = format!("{algo:?}");
|
let label = format!("{algo:?}");
|
||||||
let writer = Cursor::new(Vec::<u8>::new());
|
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
||||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||||
w.write(&[7u8; 16]).await.unwrap();
|
w.write(&[7u8; 16]).await.unwrap();
|
||||||
let written = w.into_inner().into_inner();
|
let written = w.into_inner().into_inner();
|
||||||
@@ -1492,7 +1771,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||||
w.write(payload).await.unwrap();
|
w.write(payload).await.unwrap();
|
||||||
w.into_inner().into_inner()
|
w.into_inner().into_inner()
|
||||||
}
|
}
|
||||||
@@ -1600,7 +1879,7 @@ mod tests {
|
|||||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||||
for chunk in payload.chunks(shard_size) {
|
for chunk in payload.chunks(shard_size) {
|
||||||
w.write(chunk).await.unwrap();
|
w.write(chunk).await.unwrap();
|
||||||
}
|
}
|
||||||
@@ -1674,14 +1953,14 @@ mod tests {
|
|||||||
w.write(&data).await.expect("write shard");
|
w.write(&data).await.expect("write shard");
|
||||||
|
|
||||||
let mut via_read = vec![0u8; SHARD];
|
let mut via_read = vec![0u8; SHARD];
|
||||||
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||||
.read(&mut via_read)
|
.read(&mut via_read)
|
||||||
.await
|
.await
|
||||||
.expect("read");
|
.expect("read");
|
||||||
|
|
||||||
// A buffer with only capacity — no initialized bytes at all.
|
// A buffer with only capacity — no initialized bytes at all.
|
||||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||||
.read_appending(&mut via_append, SHARD)
|
.read_appending(&mut via_append, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect("read_appending");
|
.expect("read_appending");
|
||||||
@@ -1706,7 +1985,7 @@ mod tests {
|
|||||||
encoded.truncate(encoded.len() - 1);
|
encoded.truncate(encoded.len() - 1);
|
||||||
|
|
||||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||||
.read_appending(&mut out, SHARD)
|
.read_appending(&mut out, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect_err("a truncated shard must not succeed");
|
.expect_err("a truncated shard must not succeed");
|
||||||
@@ -1732,7 +2011,7 @@ mod tests {
|
|||||||
encoded[last] ^= 0xff;
|
encoded[last] ^= 0xff;
|
||||||
|
|
||||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||||
.read_appending(&mut out, SHARD)
|
.read_appending(&mut out, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect_err("a corrupt shard must not verify");
|
.expect_err("a corrupt shard must not verify");
|
||||||
@@ -1844,7 +2123,7 @@ mod tests {
|
|||||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||||
);
|
);
|
||||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||||
let mut streamed = Cursor::new(encoded.clone());
|
let mut streamed = std::io::Cursor::new(encoded.clone());
|
||||||
assert!(
|
assert!(
|
||||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||||
"a non-Bytes source must stay on the streaming path"
|
"a non-Bytes source must stay on the streaming path"
|
||||||
@@ -1872,7 +2151,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||||
.read_appending(&mut via_stream, SHARD)
|
.read_appending(&mut via_stream, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect("streaming read");
|
.expect("streaming read");
|
||||||
|
|||||||
@@ -1116,6 +1116,14 @@ mod tests {
|
|||||||
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The lifecycle transition worker relies on this arm alone to suppress the
|
||||||
|
// closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would
|
||||||
|
// silently turn shutdown races back into `error!` log spam.
|
||||||
|
#[test]
|
||||||
|
fn is_network_or_host_down_covers_closed_network_connection() {
|
||||||
|
assert!(is_network_or_host_down("transition failed: use of closed network connection", false));
|
||||||
|
}
|
||||||
|
|
||||||
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
||||||
// every set unreachable) must NOT be classified as "all not found",
|
// every set unreachable) must NOT be classified as "all not found",
|
||||||
// otherwise ListObjects silently returns an empty listing and masks a full
|
// otherwise ListObjects silently returns an empty listing and masks a full
|
||||||
|
|||||||
@@ -704,6 +704,7 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn create_deferred_bitrot_reader(
|
pub fn create_deferred_bitrot_reader(
|
||||||
inline_data: Option<Bytes>,
|
inline_data: Option<Bytes>,
|
||||||
disk: Option<DiskStore>,
|
disk: Option<DiskStore>,
|
||||||
|
|||||||
@@ -277,6 +277,26 @@ pub struct ObjectOptions {
|
|||||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||||
pub replication_request: bool,
|
pub replication_request: bool,
|
||||||
|
/// True when the inbound request carried the
|
||||||
|
/// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
|
||||||
|
/// value "true": the request was already proxied by a replication peer,
|
||||||
|
/// so this server must not proxy a local miss onward (anti-loop,
|
||||||
|
/// MinIO-compatible). The header only disables proxying — it grants no
|
||||||
|
/// capability — so no authorization gate is required to honor it.
|
||||||
|
pub proxy_request: bool,
|
||||||
|
/// True when the `source-proxy-request` header family was present at
|
||||||
|
/// all, regardless of value (MinIO's `ProxyHeaderSet`). A replication
|
||||||
|
/// peer sends `source-proxy-request: false` on its worker convergence
|
||||||
|
/// HEADs precisely so the receiver answers locally instead of proxying
|
||||||
|
/// back — otherwise a proxied 404->200 echo makes the worker believe the
|
||||||
|
/// object already converged and it never replicates it.
|
||||||
|
pub proxy_header_set: bool,
|
||||||
|
/// Source-cluster LWW timestamps carried by an authorized replication
|
||||||
|
/// request; None when the source never modified the category. Only the
|
||||||
|
/// replication-authorized options builders may set these.
|
||||||
|
pub replication_tagging_timestamp: Option<OffsetDateTime>,
|
||||||
|
pub replication_retention_timestamp: Option<OffsetDateTime>,
|
||||||
|
pub replication_legalhold_timestamp: Option<OffsetDateTime>,
|
||||||
/// Authorized SSE-C replication passthrough: the body is already
|
/// Authorized SSE-C replication passthrough: the body is already
|
||||||
/// ciphertext, so the write path must not encrypt or compress it and
|
/// ciphertext, so the write path must not encrypt or compress it and
|
||||||
/// stores the restored encryption metadata verbatim. Only the
|
/// stores the restored encryption metadata verbatim. Only the
|
||||||
|
|||||||
@@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct DiskStat {
|
pub struct DiskStat {
|
||||||
pub total_space: u64,
|
pub total_space: u64,
|
||||||
|
|||||||
@@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize};
|
|||||||
use std::{fmt::Display, io};
|
use std::{fmt::Display, io};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "tier config wire version stamped by the parity constructors below (backlog#1823)"
|
||||||
|
)]
|
||||||
const C_TIER_CONFIG_VER: &str = "v1";
|
const C_TIER_CONFIG_VER: &str = "v1";
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)"
|
||||||
|
)]
|
||||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||||
@@ -264,7 +272,6 @@ impl Clone for TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl TierConfig {
|
impl TierConfig {
|
||||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -284,6 +291,7 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn endpoint(&self) -> String {
|
fn endpoint(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||||
@@ -303,6 +311,7 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn bucket(&self) -> String {
|
fn bucket(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||||
@@ -322,6 +331,7 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn prefix(&self) -> String {
|
fn prefix(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||||
@@ -341,6 +351,7 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn region(&self) -> String {
|
fn region(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||||
@@ -457,7 +468,7 @@ impl TierWasabi {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TierS3 {
|
impl TierS3 {
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn create<F>(
|
fn create<F>(
|
||||||
name: &str,
|
name: &str,
|
||||||
access_key: &str,
|
access_key: &str,
|
||||||
@@ -528,7 +539,7 @@ pub struct TierMinIO {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TierMinIO {
|
impl TierMinIO {
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||||
fn create<F>(
|
fn create<F>(
|
||||||
name: &str,
|
name: &str,
|
||||||
endpoint: &str,
|
endpoint: &str,
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
|
|
||||||
use crate::services::tier::tier::TierConfigMgr;
|
use crate::services::tier::tier::TierConfigMgr;
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
impl TierConfigMgr {
|
impl TierConfigMgr {
|
||||||
pub fn msg_size(&self) -> usize {
|
pub fn msg_size(&self) -> usize {
|
||||||
100
|
100
|
||||||
|
|||||||
@@ -32,15 +32,22 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
||||||
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
||||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
||||||
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
||||||
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||||
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK,
|
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
||||||
GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||||
|
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||||
|
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||||
|
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||||
|
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
||||||
|
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||||
@@ -173,11 +180,13 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
|
|||||||
Fallback(GetCodecStreamingFallbackReason),
|
Fallback(GetCodecStreamingFallbackReason),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
||||||
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MultipartCodecStreamingReader {
|
impl MultipartCodecStreamingReader {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
readers: VecDeque::from(readers),
|
readers: VecDeque::from(readers),
|
||||||
@@ -652,36 +661,15 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
|||||||
&& left.erasure.distribution == right.erasure.distribution
|
&& left.erasure.distribution == right.erasure.distribution
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
candidate: &FileInfo,
|
candidate: &FileInfo,
|
||||||
parts_metadata: &[FileInfo],
|
parts_metadata: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
) -> bool {
|
) -> Option<&'static str> {
|
||||||
if !candidate.inline_data()
|
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
|
||||||
|| candidate.is_compressed()
|
return Some(reason);
|
||||||
|| candidate
|
|
||||||
.metadata
|
|
||||||
.keys()
|
|
||||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
|
||||||
|| candidate.is_remote()
|
|
||||||
|| candidate.deleted
|
|
||||||
|| candidate.size <= 0
|
|
||||||
|| candidate.parts.len() != 1
|
|
||||||
|| !candidate.has_valid_erasure_geometry()
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
let Ok(object_size) = usize::try_from(candidate.size) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||||
@@ -690,18 +678,21 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
|||||||
candidate.erasure.block_size,
|
candidate.erasure.block_size,
|
||||||
candidate.uses_legacy_checksum,
|
candidate.uses_legacy_checksum,
|
||||||
) else {
|
) else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
};
|
};
|
||||||
let Some(data_files) =
|
let data_files =
|
||||||
collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| {
|
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
|
||||||
disks.get(index).is_some_and(Option::is_some)
|
disks.get(index).is_some_and(Option::is_some)
|
||||||
})
|
}) {
|
||||||
else {
|
Ok(data_files) => data_files,
|
||||||
return false;
|
Err(reason) => return Some(reason),
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(part) = candidate.parts.first() else {
|
let Some(part) = candidate.parts.first() else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
};
|
||||||
|
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
};
|
};
|
||||||
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
||||||
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||||
@@ -721,12 +712,111 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
|||||||
let Ok(mut readers) =
|
let Ok(mut readers) =
|
||||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||||
else {
|
else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||||
};
|
};
|
||||||
|
|
||||||
try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size)
|
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||||
.await
|
Some(body) if body.len() == object_size => None,
|
||||||
.is_some_and(|body| body.len() == object_size)
|
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
||||||
|
// `inline_data` excludes remote objects; this diagnostic reports them separately.
|
||||||
|
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
|
||||||
|
}
|
||||||
|
if candidate.is_compressed()
|
||||||
|
|| candidate
|
||||||
|
.metadata
|
||||||
|
.keys()
|
||||||
|
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||||
|
{
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED);
|
||||||
|
}
|
||||||
|
if candidate.is_remote() {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
|
||||||
|
}
|
||||||
|
if candidate.deleted {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
|
||||||
|
}
|
||||||
|
if candidate.size <= 0 {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
}
|
||||||
|
if candidate.parts.len() != 1 {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
}
|
||||||
|
if !candidate.has_valid_erasure_geometry() {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
|
}
|
||||||
|
|
||||||
|
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
};
|
||||||
|
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
}
|
||||||
|
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_read_inline_missing_shards_are_pending(
|
||||||
|
candidate: &FileInfo,
|
||||||
|
parts_metadata: &[FileInfo],
|
||||||
|
errors: &[Option<DiskError>],
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
fanout_order: &[usize],
|
||||||
|
scheduled_fanout_len: usize,
|
||||||
|
) -> bool {
|
||||||
|
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||||
|
candidate.erasure.data_blocks,
|
||||||
|
candidate.erasure.parity_blocks,
|
||||||
|
candidate.erasure.block_size,
|
||||||
|
candidate.uses_legacy_checksum,
|
||||||
|
) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let distribution = &candidate.erasure.distribution;
|
||||||
|
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
|
||||||
|
let mut missing_pending_data_shards = 0usize;
|
||||||
|
|
||||||
|
for (disk_index, file_info) in parts_metadata.iter().enumerate() {
|
||||||
|
let Some(&block_index) = distribution.get(disk_index) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if block_index == 0 || block_index > erasure.data_shards {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !disks.get(disk_index).is_some_and(Option::is_some) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data_slot = block_index - 1;
|
||||||
|
if file_info.name.is_empty() {
|
||||||
|
let scheduled_and_not_failed = fanout_order
|
||||||
|
.get(..scheduled_fanout_len)
|
||||||
|
.is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index))
|
||||||
|
&& errors.get(disk_index).is_some_and(Option::is_none);
|
||||||
|
if scheduled_and_not_failed {
|
||||||
|
data_shards_seen_or_pending[data_slot] = true;
|
||||||
|
missing_pending_data_shards = missing_pending_data_shards.saturating_add(1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if file_info.erasure.index != block_index
|
||||||
|
|| !file_info.has_valid_erasure_geometry()
|
||||||
|
|| !metadata_early_stop_candidate_matches(file_info, candidate)
|
||||||
|
|| file_info.data.as_ref().is_none_or(|data| data.is_empty())
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
data_shards_seen_or_pending[data_slot] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
||||||
@@ -1758,6 +1848,7 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
||||||
files: &[FileInfo],
|
files: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
@@ -2048,6 +2139,7 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
|
|||||||
setup
|
setup
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
||||||
tasks: Vec<F>,
|
tasks: Vec<F>,
|
||||||
read_quorum: usize,
|
read_quorum: usize,
|
||||||
@@ -2364,6 +2456,7 @@ impl SetDisks {
|
|||||||
let bucket: Arc<str> = Arc::from(bucket);
|
let bucket: Arc<str> = Arc::from(bucket);
|
||||||
let object: Arc<str> = Arc::from(object);
|
let object: Arc<str> = Arc::from(object);
|
||||||
let version_id: Arc<str> = Arc::from(version_id);
|
let version_id: Arc<str> = Arc::from(version_id);
|
||||||
|
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||||
let disk = disk.clone();
|
let disk = disk.clone();
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2371,10 +2464,14 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
|
let slowtail_fault = slowtail_fault.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let response_start = observe.then(Instant::now);
|
let response_start = observe.then(Instant::now);
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
Self::record_read_version_call(&object, disk_index);
|
Self::record_read_version_call(&object, disk_index);
|
||||||
|
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2469,6 +2566,8 @@ impl SetDisks {
|
|||||||
let mut next_fanout_index = 0usize;
|
let mut next_fanout_index = 0usize;
|
||||||
let mut scheduled_count = 0usize;
|
let mut scheduled_count = 0usize;
|
||||||
let mut force_full_wait = false;
|
let mut force_full_wait = false;
|
||||||
|
let mut final_miss_reason_override = None;
|
||||||
|
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||||
let spawn_read_version =
|
let spawn_read_version =
|
||||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2476,6 +2575,7 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
|
let slowtail_fault = slowtail_fault.clone();
|
||||||
join_set.spawn(async move {
|
join_set.spawn(async move {
|
||||||
let response_start = Instant::now();
|
let response_start = Instant::now();
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
@@ -2484,6 +2584,9 @@ impl SetDisks {
|
|||||||
Self::record_read_version_call(&object, index);
|
Self::record_read_version_call(&object, index);
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
Self::read_version_fanout_barrier(&object, index).await;
|
Self::read_version_fanout_barrier(&object, index).await;
|
||||||
|
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2511,11 +2614,20 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
while let Some(result) = join_set.join_next().await {
|
while let Some(result) = join_set.join_next().await {
|
||||||
|
let mut defer_pending_inline_data_shard = false;
|
||||||
match result {
|
match result {
|
||||||
Ok((index, res, elapsed)) => match res {
|
Ok((index, res, elapsed)) => match res {
|
||||||
Ok(file_info) => {
|
Ok(file_info) => {
|
||||||
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
||||||
accumulator.observe_file_info(&file_info);
|
accumulator.observe_file_info(&file_info);
|
||||||
|
if bounded_fanout
|
||||||
|
&& read_data
|
||||||
|
&& !force_full_wait
|
||||||
|
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
|
||||||
|
{
|
||||||
|
force_full_wait = true;
|
||||||
|
final_miss_reason_override.get_or_insert(reason);
|
||||||
|
}
|
||||||
if let Some(slot) = ress.get_mut(index) {
|
if let Some(slot) = ress.get_mut(index) {
|
||||||
*slot = file_info;
|
*slot = file_info;
|
||||||
}
|
}
|
||||||
@@ -2541,17 +2653,43 @@ impl SetDisks {
|
|||||||
.or_else(|| accumulator.version_early_stop_decision())
|
.or_else(|| accumulator.version_early_stop_decision())
|
||||||
{
|
{
|
||||||
let should_return_early = if read_data {
|
let should_return_early = if read_data {
|
||||||
let allow_data_read_early_stop = match accumulator.candidate.as_ref() {
|
match accumulator.candidate.as_ref() {
|
||||||
Some(candidate) => {
|
Some(candidate) => match data_read_early_stop_inline_body_miss_reason(
|
||||||
data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks)
|
bucket.as_ref(),
|
||||||
.await
|
object.as_ref(),
|
||||||
|
candidate,
|
||||||
|
&ress,
|
||||||
|
disks,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
None => true,
|
||||||
|
Some(reason) => {
|
||||||
|
final_miss_reason_override = Some(reason);
|
||||||
|
if bounded_fanout
|
||||||
|
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
|
||||||
|
&& data_read_inline_missing_shards_are_pending(
|
||||||
|
candidate,
|
||||||
|
&ress,
|
||||||
|
&errors,
|
||||||
|
disks,
|
||||||
|
&fanout_order,
|
||||||
|
next_fanout_index,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
defer_pending_inline_data_shard = true;
|
||||||
|
} else {
|
||||||
|
force_full_wait = true;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
None => {
|
||||||
|
force_full_wait = true;
|
||||||
|
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
|
||||||
|
false
|
||||||
}
|
}
|
||||||
None => false,
|
|
||||||
};
|
|
||||||
if !allow_data_read_early_stop {
|
|
||||||
force_full_wait = true;
|
|
||||||
}
|
}
|
||||||
allow_data_read_early_stop
|
|
||||||
} else {
|
} else {
|
||||||
true
|
true
|
||||||
};
|
};
|
||||||
@@ -2588,6 +2726,7 @@ impl SetDisks {
|
|||||||
let pending_responses = join_set.len();
|
let pending_responses = join_set.len();
|
||||||
let should_hedge_single_pending_data_read = read_data
|
let should_hedge_single_pending_data_read = read_data
|
||||||
&& !force_full_wait
|
&& !force_full_wait
|
||||||
|
&& !defer_pending_inline_data_shard
|
||||||
&& pending_responses == 1
|
&& pending_responses == 1
|
||||||
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
||||||
if bounded_fanout && force_full_wait {
|
if bounded_fanout && force_full_wait {
|
||||||
@@ -2600,6 +2739,7 @@ impl SetDisks {
|
|||||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||||
}
|
}
|
||||||
} else if bounded_fanout
|
} else if bounded_fanout
|
||||||
|
&& !defer_pending_inline_data_shard
|
||||||
&& next_fanout_index < disks.len()
|
&& next_fanout_index < disks.len()
|
||||||
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||||
|| should_hedge_single_pending_data_read)
|
|| should_hedge_single_pending_data_read)
|
||||||
@@ -2613,7 +2753,12 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason());
|
let accumulator_miss_reason = accumulator.final_miss_reason();
|
||||||
|
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
|
||||||
|
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
|
||||||
|
_ => accumulator_miss_reason,
|
||||||
|
};
|
||||||
|
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
||||||
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
||||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||||
@@ -2842,6 +2987,7 @@ impl SetDisks {
|
|||||||
(meta_file_infos, errs)
|
(meta_file_infos, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn read_multiple_files(
|
pub(in crate::set_disk) async fn read_multiple_files(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
req: ReadMultipleReq,
|
req: ReadMultipleReq,
|
||||||
@@ -2875,14 +3021,11 @@ impl SetDisks {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||||
Ok(collected) => collected,
|
Ok(collected) => collected,
|
||||||
Err(()) => return empty_quorum_result(),
|
Err(()) => return empty_quorum_result(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
|
||||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
|
||||||
|
|
||||||
let mut ret = Vec::with_capacity(req.files.len());
|
let mut ret = Vec::with_capacity(req.files.len());
|
||||||
|
|
||||||
for want in req.files.iter() {
|
for want in req.files.iter() {
|
||||||
@@ -3021,6 +3164,7 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
|||||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
type RenameDataLegacyTuple = (
|
type RenameDataLegacyTuple = (
|
||||||
Vec<Option<DiskStore>>,
|
Vec<Option<DiskStore>>,
|
||||||
RenameConvergence,
|
RenameConvergence,
|
||||||
@@ -3030,6 +3174,7 @@ type RenameDataLegacyTuple = (
|
|||||||
);
|
);
|
||||||
|
|
||||||
impl RenameDataCommit {
|
impl RenameDataCommit {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
||||||
(
|
(
|
||||||
self.online_disks,
|
self.online_disks,
|
||||||
@@ -3148,6 +3293,7 @@ impl SetDisks {
|
|||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||||
#[allow(clippy::type_complexity)]
|
#[allow(clippy::type_complexity)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn rename_data(
|
pub(in crate::set_disk) async fn rename_data(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
@@ -4714,6 +4860,14 @@ impl SetDisks {
|
|||||||
/// is best-effort maintenance: individual delete failures are logged and
|
/// is best-effort maintenance: individual delete failures are logged and
|
||||||
/// skipped rather than propagated.
|
/// skipped rather than propagated.
|
||||||
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||||
|
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||||
|
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
||||||
let disks = self.get_disks_internal().await;
|
let disks = self.get_disks_internal().await;
|
||||||
|
|
||||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||||
@@ -4821,6 +4975,20 @@ impl SetDisks {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let stray = format!("{object}/{dir}");
|
let stray = format!("{object}/{dir}");
|
||||||
|
if dry_run {
|
||||||
|
removed += 1;
|
||||||
|
debug!(
|
||||||
|
target: "rustfs_ecstore::set_disk",
|
||||||
|
event = "heal_abandoned_parts",
|
||||||
|
component = "ecstore",
|
||||||
|
subsystem = "heal",
|
||||||
|
state = "dry_run_matched",
|
||||||
|
result = "matched",
|
||||||
|
bucket, object, data_dir = %dir,
|
||||||
|
"Heal abandoned parts dry-run matched orphaned data directory"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
match disk
|
match disk
|
||||||
.delete(
|
.delete(
|
||||||
bucket,
|
bucket,
|
||||||
@@ -4960,6 +5128,7 @@ fn is_cleanup_not_found(e: &DiskError) -> bool {
|
|||||||
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
||||||
/// must not be silently swallowed as an ignorable error (fixes the historical
|
/// must not be silently swallowed as an ignorable error (fixes the historical
|
||||||
/// `Unexpected`/`DiskNotFound` misclassification).
|
/// `Unexpected`/`DiskNotFound` misclassification).
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
||||||
match joined {
|
match joined {
|
||||||
Ok(res) => res,
|
Ok(res) => res,
|
||||||
@@ -5184,6 +5353,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
|||||||
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
||||||
pub const CLEANUP: &str = "cleanup";
|
pub const CLEANUP: &str = "cleanup";
|
||||||
/// The per-disk `read_version` phase of metadata read fan-out.
|
/// The per-disk `read_version` phase of metadata read fan-out.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub const READ_VERSION: &str = "read_version";
|
pub const READ_VERSION: &str = "read_version";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5621,6 +5791,130 @@ mod tests {
|
|||||||
(dirs, disks)
|
(dirs, disks)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert_eq!(
|
||||||
|
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
|
||||||
|
Some(Duration::from_millis(25))
|
||||||
|
);
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
|
||||||
|
const DISKS: usize = 4;
|
||||||
|
let bucket = "metadata-slowtail-fault-bucket";
|
||||||
|
let object = "objects/metadata-slowtail-fault-object";
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let read_without_data =
|
||||||
|
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
|
||||||
|
tokio::time::timeout(Duration::from_millis(100), read_without_data)
|
||||||
|
.await
|
||||||
|
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
|
||||||
|
.expect("metadata fanout without read_data should resolve");
|
||||||
|
|
||||||
|
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"data-read metadata fanout must wait for the injected slow read_version response"
|
||||||
|
);
|
||||||
|
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||||
|
.await
|
||||||
|
.expect("injected slowtail should eventually complete")
|
||||||
|
.expect("data-read metadata fanout should resolve");
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
|
||||||
|
const DISKS: usize = 4;
|
||||||
|
let bucket = "metadata-slowtail-early-stop-bucket";
|
||||||
|
let object = "objects/metadata-slowtail-early-stop-object";
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
|
||||||
|
);
|
||||||
|
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||||
|
.await
|
||||||
|
.expect("injected early-stop slowtail should eventually complete")
|
||||||
|
.expect("early-stop metadata fanout should resolve");
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||||
///
|
///
|
||||||
/// The metadata fan-out issues each `read_version` inside its own
|
/// The metadata fan-out issues each `read_version` inside its own
|
||||||
@@ -5752,9 +6046,20 @@ mod tests {
|
|||||||
object: &str,
|
object: &str,
|
||||||
payload: &[u8],
|
payload: &[u8],
|
||||||
uses_legacy_checksum: bool,
|
uses_legacy_checksum: bool,
|
||||||
|
) -> Vec<FileInfo> {
|
||||||
|
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn inline_metadata_fanout_fileinfos_with_geometry(
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
payload: &[u8],
|
||||||
|
uses_legacy_checksum: bool,
|
||||||
|
data_shards: usize,
|
||||||
|
parity_shards: usize,
|
||||||
) -> Vec<FileInfo> {
|
) -> Vec<FileInfo> {
|
||||||
let distribution_key = metadata_distribution_key(bucket, object);
|
let distribution_key = metadata_distribution_key(bucket, object);
|
||||||
let mut base = FileInfo::new(&distribution_key, 2, 2);
|
let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards);
|
||||||
base.volume = bucket.to_string();
|
base.volume = bucket.to_string();
|
||||||
base.name = object.to_string();
|
base.name = object.to_string();
|
||||||
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
||||||
@@ -5817,6 +6122,21 @@ mod tests {
|
|||||||
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn install_inline_metadata_fanout_fileinfo_with_geometry(
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
payload: &[u8],
|
||||||
|
data_shards: usize,
|
||||||
|
parity_shards: usize,
|
||||||
|
mutate: impl FnOnce(&mut [FileInfo]),
|
||||||
|
) {
|
||||||
|
let mut files =
|
||||||
|
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await;
|
||||||
|
mutate(&mut files);
|
||||||
|
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
||||||
|
}
|
||||||
|
|
||||||
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
||||||
let distribution = files
|
let distribution = files
|
||||||
.first()
|
.first()
|
||||||
@@ -6037,6 +6357,118 @@ mod tests {
|
|||||||
drop(dirs);
|
drop(dirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() {
|
||||||
|
const DISKS: usize = 6;
|
||||||
|
const DATA_SHARDS: usize = 4;
|
||||||
|
const PARITY_SHARDS: usize = 2;
|
||||||
|
let bucket = "bounded-inline-data-get-pending-shard-bucket";
|
||||||
|
let object =
|
||||||
|
object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS);
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_inline_metadata_fanout_fileinfo_with_geometry(
|
||||||
|
&disks,
|
||||||
|
bucket,
|
||||||
|
&object,
|
||||||
|
b"verified inline payload",
|
||||||
|
DATA_SHARDS,
|
||||||
|
PARITY_SHARDS,
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||||
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||||
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS);
|
||||||
|
let distribution_key = metadata_distribution_key(bucket, &object);
|
||||||
|
let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS)
|
||||||
|
.erasure
|
||||||
|
.distribution;
|
||||||
|
let paused_data_disk = *fanout_order
|
||||||
|
.iter()
|
||||||
|
.take(DATA_SHARDS)
|
||||||
|
.find(|disk_index| {
|
||||||
|
distribution
|
||||||
|
.get(**disk_index)
|
||||||
|
.is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index))
|
||||||
|
})
|
||||||
|
.expect("initial fanout should include a data shard to pause");
|
||||||
|
let hedged_parity_disk = fanout_order[DATA_SHARDS];
|
||||||
|
let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1];
|
||||||
|
|
||||||
|
let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
|
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
||||||
|
let calls = disk_call_counters::observe(&object);
|
||||||
|
let disks_for_read = disks.clone();
|
||||||
|
let object_for_read = object.clone();
|
||||||
|
let mut read = tokio::spawn(async move {
|
||||||
|
SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks_for_read,
|
||||||
|
bucket,
|
||||||
|
bucket,
|
||||||
|
&object_for_read,
|
||||||
|
"",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
PARITY_SHARDS,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||||
|
.await
|
||||||
|
.expect("initial data shard should pause before returning");
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||||
|
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded fanout should hedge one parity disk while the data shard is pending");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
|
||||||
|
"inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait"
|
||||||
|
);
|
||||||
|
|
||||||
|
barrier.release();
|
||||||
|
let (parts_metadata, errs, diagnostics) = read
|
||||||
|
.await
|
||||||
|
.expect("metadata read task should not panic")
|
||||||
|
.expect("pending data shard should let the inline verifier finish");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
|
5,
|
||||||
|
"pending data-shard defer should not schedule the final parity disk"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk),
|
||||||
|
0,
|
||||||
|
"the remaining parity disk must stay unissued when pending data verification succeeds"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tracker.running(),
|
||||||
|
0,
|
||||||
|
"early-stop should drain spawned read_version tasks before returning"
|
||||||
|
);
|
||||||
|
assert_eq!(diagnostics.total_responses(), 5);
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
||||||
let bucket = "legacy-inline-data-get-fanout-bucket";
|
let bucket = "legacy-inline-data-get-fanout-bucket";
|
||||||
@@ -6067,11 +6499,133 @@ mod tests {
|
|||||||
.clone();
|
.clone();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await,
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks)
|
||||||
|
.await
|
||||||
|
.is_none(),
|
||||||
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn data_read_early_stop_reports_inline_miss_reasons() {
|
||||||
|
let bucket = "inline-data-get-miss-reason-bucket";
|
||||||
|
let object = "inline-data-get-miss-reason-object";
|
||||||
|
let payload = b"verified inline payload";
|
||||||
|
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
|
||||||
|
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
|
||||||
|
let distribution = files
|
||||||
|
.first()
|
||||||
|
.map(|file| file.erasure.distribution.clone())
|
||||||
|
.expect("fixture should include metadata");
|
||||||
|
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
|
||||||
|
let mut parts_metadata = vec![FileInfo::default(); 4];
|
||||||
|
for disk_index in order.into_iter().take(3) {
|
||||||
|
let block_index = distribution
|
||||||
|
.get(disk_index)
|
||||||
|
.copied()
|
||||||
|
.expect("fixture distribution should cover every disk");
|
||||||
|
parts_metadata[disk_index] = files
|
||||||
|
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
|
||||||
|
.expect("fixture should include every distributed shard")
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
let candidate = parts_metadata
|
||||||
|
.iter()
|
||||||
|
.find(|file| file.name == object)
|
||||||
|
.expect("fixture should include observed metadata")
|
||||||
|
.clone();
|
||||||
|
let data_disk = distribution
|
||||||
|
.iter()
|
||||||
|
.position(|block_index| *block_index == 1)
|
||||||
|
.expect("fixture distribution should include first data shard");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut not_inline = candidate.clone();
|
||||||
|
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, ¬_inline, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut remote = candidate.clone();
|
||||||
|
remote.transition_status = TRANSITION_COMPLETE.to_string();
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut transformed = candidate.clone();
|
||||||
|
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut deleted = candidate.clone();
|
||||||
|
deleted.deleted = true;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut zero_size = candidate.clone();
|
||||||
|
zero_size.size = 0;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut multipart = candidate.clone();
|
||||||
|
multipart.parts.push(multipart.parts[0].clone());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut invalid_geometry = candidate.clone();
|
||||||
|
invalid_geometry.erasure.data_blocks = 0;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut missing_shard = parts_metadata.clone();
|
||||||
|
missing_shard[data_disk] = FileInfo::default();
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut missing_payload = parts_metadata.clone();
|
||||||
|
missing_payload[data_disk].data = None;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut identity_mismatch = parts_metadata.clone();
|
||||||
|
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut corrupt = parts_metadata.clone();
|
||||||
|
if let Some(data) = corrupt[data_disk].data.as_mut() {
|
||||||
|
let mut corrupt_data = data.to_vec();
|
||||||
|
corrupt_data[0] ^= 0x01;
|
||||||
|
*data = Bytes::from(corrupt_data);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
||||||
@@ -6161,7 +6715,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_INTERNAL_META),
|
("path", GET_OBJECT_PATH_INTERNAL_META),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
1,
|
1,
|
||||||
@@ -6173,7 +6727,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
0,
|
0,
|
||||||
@@ -6708,7 +7262,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() {
|
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-hedge-bucket";
|
let bucket = "bounded-data-get-hedge-bucket";
|
||||||
let object = "bounded-data-get-hedge-object";
|
let object = "bounded-data-get-hedge-object";
|
||||||
@@ -6739,7 +7293,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("bounded data-read fanout should hedge by starting the spare disk");
|
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk");
|
||||||
|
|
||||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||||
assert!(
|
assert!(
|
||||||
@@ -6755,7 +7309,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"bounded data-read fanout should issue the paused disk plus one spare hedge"
|
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
@@ -6768,7 +7322,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
|
async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-default-bucket";
|
let bucket = "bounded-data-get-default-bucket";
|
||||||
let object = "bounded-data-get-default-object";
|
let object = "bounded-data-get-default-object";
|
||||||
@@ -6782,16 +7336,42 @@ mod tests {
|
|||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
|
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
let calls = disk_call_counters::observe(object);
|
let calls = disk_call_counters::observe(object);
|
||||||
let (parts_metadata, errs, diagnostics) =
|
let disks_for_read = disks.clone();
|
||||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
|
let mut read = tokio::spawn(async move {
|
||||||
|
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
|
||||||
.await
|
.await
|
||||||
.expect("default data-read metadata should resolve");
|
});
|
||||||
|
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||||
|
.await
|
||||||
|
.expect("default bounded non-inline read should schedule the paused metadata task");
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||||
|
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect(
|
||||||
|
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
|
||||||
|
);
|
||||||
|
|
||||||
|
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||||
|
assert!(
|
||||||
|
pending.is_err(),
|
||||||
|
"default non-inline data reads must not return before the paused metadata response"
|
||||||
|
);
|
||||||
|
barrier.release();
|
||||||
|
let (parts_metadata, errs, diagnostics) = read
|
||||||
|
.await
|
||||||
|
.expect("metadata read task should not panic")
|
||||||
|
.expect("default data-read metadata should resolve");
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"default GET data-read metadata must keep full fanout for read-failure tolerance"
|
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
|||||||
@@ -42,12 +42,20 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The borrowed core, for state not yet fronted by a typed accessor.
|
/// The borrowed core, for state not yet fronted by a typed accessor.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn core(&self) -> &'a SetDisks {
|
pub(crate) fn core(&self) -> &'a SetDisks {
|
||||||
self.core
|
self.core
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Immutable topology / config (fixed after construction) ---
|
// --- Immutable topology / config (fixed after construction) ---
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_index(&self) -> usize {
|
pub(crate) fn set_index(&self) -> usize {
|
||||||
self.core.set_index
|
self.core.set_index
|
||||||
}
|
}
|
||||||
@@ -56,14 +64,26 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
self.core.pool_index
|
self.core.pool_index
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_drive_count(&self) -> usize {
|
pub(crate) fn set_drive_count(&self) -> usize {
|
||||||
self.core.set_drive_count
|
self.core.set_drive_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn default_parity_count(&self) -> usize {
|
pub(crate) fn default_parity_count(&self) -> usize {
|
||||||
self.core.default_parity_count
|
self.core.default_parity_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
||||||
&self.core.set_endpoints
|
&self.core.set_endpoints
|
||||||
}
|
}
|
||||||
@@ -72,6 +92,10 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
&self.core.format
|
&self.core.format
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn locker_owner(&self) -> &'a str {
|
pub(crate) fn locker_owner(&self) -> &'a str {
|
||||||
&self.core.locker_owner
|
&self.core.locker_owner
|
||||||
}
|
}
|
||||||
@@ -84,6 +108,10 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
|
|
||||||
// --- Locker trio ---
|
// --- Locker trio ---
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
||||||
&self.core.lockers
|
&self.core.lockers
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user