mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4ea0934d45 | |||
| 3c88e8df8a |
@@ -15,35 +15,28 @@
|
|||||||
# Package Workflow - Build DEB/RPM packages
|
# Package Workflow - Build DEB/RPM packages
|
||||||
#
|
#
|
||||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
# and uploads them to Cloudflare R2.
|
||||||
#
|
#
|
||||||
# Trigger:
|
# Trigger:
|
||||||
# - workflow_run: automatically package after "Build and Release" completes
|
# - release published: automatically package when a GitHub release is published
|
||||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||||
# to the GitHub release before packaging starts)
|
|
||||||
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
|
||||||
#
|
#
|
||||||
# Flow:
|
# Flow:
|
||||||
# 1. Resolve the triggering Build workflow run for the release tag
|
# 1. Find the Build workflow run for the release tag
|
||||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||||
# 3. Build DEB packages for amd64 and arm64
|
# 3. Build DEB packages for amd64 and arm64
|
||||||
# 4. Build RPM packages for x86_64 and aarch64
|
# 4. Build RPM packages for x86_64 and aarch64
|
||||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
# 5. Upload all packages to Cloudflare R2
|
||||||
|
|
||||||
name: Package DEB/RPM
|
name: Package DEB/RPM
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
# contents: write is required to upload packages to the GitHub release
|
contents: read
|
||||||
contents: write
|
|
||||||
actions: read
|
actions: read
|
||||||
|
|
||||||
on:
|
on:
|
||||||
# Follows the same pattern as docker.yml: run after the release build
|
release:
|
||||||
# workflow completes, so packaging is triggered only by release tags
|
types: [ published ]
|
||||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
|
||||||
workflow_run:
|
|
||||||
workflows: [ "Build and Release" ]
|
|
||||||
types: [ completed ]
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
tag:
|
tag:
|
||||||
@@ -56,26 +49,13 @@ on:
|
|||||||
type: string
|
type: string
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
env:
|
|
||||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
|
||||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Resolve which build run to use and extract version info
|
# Resolve which build run to use and extract version info
|
||||||
resolve:
|
resolve:
|
||||||
name: Resolve Build
|
name: Resolve Build
|
||||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
|
||||||
# Tag pushes arrive as event == push with head_branch != main (a
|
|
||||||
# non-main push head_branch is the release tag name). Manual dispatch
|
|
||||||
# stays available as a fallback for backfills and re-runs.
|
|
||||||
if: >-
|
|
||||||
github.event_name == 'workflow_dispatch' ||
|
|
||||||
(github.event.workflow_run.conclusion == 'success' &&
|
|
||||||
github.event.workflow_run.event == 'push' &&
|
|
||||||
github.event.workflow_run.head_branch != 'main')
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
outputs:
|
outputs:
|
||||||
@@ -95,8 +75,8 @@ jobs:
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Determine tag
|
# Determine tag
|
||||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||||
TAG="${HEAD_BRANCH}"
|
TAG="${{ github.event.release.tag_name }}"
|
||||||
elif [[ -n "$INPUT_TAG" ]]; then
|
elif [[ -n "$INPUT_TAG" ]]; then
|
||||||
TAG="$INPUT_TAG"
|
TAG="$INPUT_TAG"
|
||||||
else
|
else
|
||||||
@@ -113,11 +93,6 @@ jobs:
|
|||||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||||
|
|
||||||
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
|
||||||
# Use the Build and Release run that triggered this workflow
|
|
||||||
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
|
||||||
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
|
||||||
|
|
||||||
elif [[ -n "$TAG" ]]; then
|
elif [[ -n "$TAG" ]]; then
|
||||||
# Find the build run that produced this tag
|
# Find the build run that produced this tag
|
||||||
echo "Looking for build run for tag: $TAG"
|
echo "Looking for build run for tag: $TAG"
|
||||||
@@ -481,54 +456,6 @@ jobs:
|
|||||||
echo "✅ Latest packages updated"
|
echo "✅ Latest packages updated"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload packages to GitHub Release
|
|
||||||
if: needs.resolve.outputs.tag != ''
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ github.token }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
TAG="${{ needs.resolve.outputs.tag }}"
|
|
||||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
|
||||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
|
||||||
|
|
||||||
# Upload the packages, then refresh the release checksums so the new
|
|
||||||
# assets are covered, matching the binary release flow.
|
|
||||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
|
||||||
if [[ -n "$f" && -f "$f" ]]; then
|
|
||||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
|
||||||
gh release upload "$TAG" "$f" --clobber
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
CHECKSUM_DIR="$(mktemp -d)"
|
|
||||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
|
||||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
|
||||||
|
|
||||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
|
||||||
asset="${spec%%:*}"
|
|
||||||
checksum_cmd="${spec##*:}"
|
|
||||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
|
||||||
|
|
||||||
touch "$checksum_file"
|
|
||||||
|
|
||||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
|
||||||
if [[ -n "$f" && -f "$f" ]]; then
|
|
||||||
base="$(basename "$f")"
|
|
||||||
# Remove any stale entry, then append the fresh digest
|
|
||||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
|
||||||
mv "${checksum_file}.tmp" "$checksum_file"
|
|
||||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
|
||||||
gh release upload "$TAG" "$checksum_file" --clobber
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "✅ GitHub release assets updated"
|
|
||||||
|
|
||||||
# Summary
|
# Summary
|
||||||
summary:
|
summary:
|
||||||
name: Summary
|
name: Summary
|
||||||
|
|||||||
Generated
+103
-115
@@ -278,7 +278,6 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bigdecimal",
|
"bigdecimal",
|
||||||
"bon",
|
"bon",
|
||||||
"crc32fast",
|
|
||||||
"digest 0.11.3",
|
"digest 0.11.3",
|
||||||
"log",
|
"log",
|
||||||
"miniz_oxide 0.9.1",
|
"miniz_oxide 0.9.1",
|
||||||
@@ -290,11 +289,9 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"snap",
|
|
||||||
"strum",
|
"strum",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
"uuid",
|
"uuid",
|
||||||
"zstd",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -964,9 +961,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-kms"
|
name = "aws-sdk-kms"
|
||||||
version = "1.115.0"
|
version = "1.114.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
|
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -990,9 +987,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-s3"
|
name = "aws-sdk-s3"
|
||||||
version = "1.142.0"
|
version = "1.141.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
|
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1027,9 +1024,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-sso"
|
name = "aws-sdk-sso"
|
||||||
version = "1.106.0"
|
version = "1.105.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
|
checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1053,9 +1050,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-ssooidc"
|
name = "aws-sdk-ssooidc"
|
||||||
version = "1.108.0"
|
version = "1.107.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
|
checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1079,9 +1076,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-sdk-sts"
|
name = "aws-sdk-sts"
|
||||||
version = "1.111.0"
|
version = "1.110.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
|
checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"aws-credential-types",
|
"aws-credential-types",
|
||||||
@@ -1598,7 +1595,7 @@ version = "0.10.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1617,7 +1614,7 @@ version = "0.3.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1858,9 +1855,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.4.3"
|
version = "1.4.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
"jobserver",
|
"jobserver",
|
||||||
@@ -1968,7 +1965,7 @@ version = "0.4.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crypto-common 0.1.7",
|
"crypto-common 0.1.6",
|
||||||
"inout 0.1.4",
|
"inout 0.1.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2428,7 +2425,7 @@ version = "0.5.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -2453,11 +2450,11 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.7"
|
version = "0.1.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3664,7 +3661,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer 0.10.4",
|
"block-buffer 0.10.4",
|
||||||
"const-oid 0.9.6",
|
"const-oid 0.9.6",
|
||||||
"crypto-common 0.1.7",
|
"crypto-common 0.1.6",
|
||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3924,7 +3921,7 @@ dependencies = [
|
|||||||
"crypto-bigint 0.5.5",
|
"crypto-bigint 0.5.5",
|
||||||
"digest 0.10.7",
|
"digest 0.10.7",
|
||||||
"ff 0.13.1",
|
"ff 0.13.1",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"group 0.13.0",
|
"group 0.13.0",
|
||||||
"hkdf 0.12.4",
|
"hkdf 0.12.4",
|
||||||
"pem-rfc7468 0.7.0",
|
"pem-rfc7468 0.7.0",
|
||||||
@@ -4148,9 +4145,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.11"
|
version = "0.1.10"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
|
checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "findshlibs"
|
name = "findshlibs"
|
||||||
@@ -4369,9 +4366,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.7"
|
version = "0.14.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"typenum",
|
"typenum",
|
||||||
"version_check",
|
"version_check",
|
||||||
@@ -4380,11 +4377,11 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "1.4.5"
|
version = "1.4.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"rustversion",
|
"rustversion",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
@@ -4726,9 +4723,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.4.16"
|
version = "0.4.15"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atomic-waker",
|
"atomic-waker",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -5028,9 +5025,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath"
|
name = "hotpath"
|
||||||
version = "0.23.3"
|
version = "0.23.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
|
checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-channel",
|
"async-channel",
|
||||||
@@ -5062,9 +5059,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-macros"
|
name = "hotpath-macros"
|
||||||
version = "0.23.3"
|
version = "0.23.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
|
checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
@@ -5073,15 +5070,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-macros-meta"
|
name = "hotpath-macros-meta"
|
||||||
version = "0.23.3"
|
version = "0.23.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
|
checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hotpath-meta"
|
name = "hotpath-meta"
|
||||||
version = "0.23.3"
|
version = "0.23.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
|
checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hotpath-macros-meta",
|
"hotpath-macros-meta",
|
||||||
]
|
]
|
||||||
@@ -5280,9 +5277,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_collections"
|
name = "icu_collections"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
|
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"potential_utf",
|
"potential_utf",
|
||||||
@@ -5294,9 +5291,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_locale_core"
|
name = "icu_locale_core"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
|
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"litemap",
|
"litemap",
|
||||||
@@ -5307,9 +5304,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_normalizer"
|
name = "icu_normalizer"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
|
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"icu_collections",
|
"icu_collections",
|
||||||
"icu_normalizer_data",
|
"icu_normalizer_data",
|
||||||
@@ -5321,17 +5318,16 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_normalizer_data"
|
name = "icu_normalizer_data"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
|
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_properties"
|
name = "icu_properties"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
|
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
|
||||||
"icu_collections",
|
"icu_collections",
|
||||||
"icu_locale_core",
|
"icu_locale_core",
|
||||||
"icu_properties_data",
|
"icu_properties_data",
|
||||||
@@ -5342,15 +5338,15 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_properties_data"
|
name = "icu_properties_data"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
|
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "icu_provider"
|
name = "icu_provider"
|
||||||
version = "2.3.0"
|
version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
|
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"icu_locale_core",
|
"icu_locale_core",
|
||||||
@@ -5418,7 +5414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"block-padding 0.3.3",
|
"block-padding 0.3.3",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5969,9 +5965,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libredox"
|
name = "libredox"
|
||||||
version = "0.1.20"
|
version = "0.1.19"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a"
|
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
@@ -6034,9 +6030,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "litemap"
|
name = "litemap"
|
||||||
version = "0.8.3"
|
version = "0.8.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "local-ip-address"
|
name = "local-ip-address"
|
||||||
@@ -6483,9 +6479,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mqttbytes-core-next"
|
name = "mqttbytes-core-next"
|
||||||
version = "0.34.0"
|
version = "0.33.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa"
|
checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
@@ -6886,7 +6882,7 @@ version = "5.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.22.1",
|
"base64 0.21.7",
|
||||||
"chrono",
|
"chrono",
|
||||||
"getrandom 0.2.17",
|
"getrandom 0.2.17",
|
||||||
"http 1.5.0",
|
"http 1.5.0",
|
||||||
@@ -7345,9 +7341,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pageant"
|
name = "pageant"
|
||||||
version = "0.2.2"
|
version = "0.2.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
|
checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base16ct 1.0.0",
|
"base16ct 1.0.0",
|
||||||
"byteorder",
|
"byteorder",
|
||||||
@@ -7729,9 +7725,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pkg-config"
|
name = "pkg-config"
|
||||||
version = "0.3.34"
|
version = "0.3.33"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "plotters"
|
name = "plotters"
|
||||||
@@ -7837,9 +7833,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "potential_utf"
|
name = "potential_utf"
|
||||||
version = "0.1.6"
|
version = "0.1.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
|
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"zerovec",
|
"zerovec",
|
||||||
]
|
]
|
||||||
@@ -8047,7 +8043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"itertools 0.14.0",
|
"itertools 0.10.5",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
@@ -8067,7 +8063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"itertools 0.14.0",
|
"itertools 0.10.5",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"petgraph 0.8.3",
|
"petgraph 0.8.3",
|
||||||
@@ -8088,7 +8084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.14.0",
|
"itertools 0.10.5",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
@@ -8101,7 +8097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.14.0",
|
"itertools 0.10.5",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
@@ -8217,7 +8213,7 @@ dependencies = [
|
|||||||
"reqwest",
|
"reqwest",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"spin 0.12.3",
|
"spin 0.12.2",
|
||||||
"symbolic-demangle",
|
"symbolic-demangle",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
@@ -8286,9 +8282,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quinn-proto"
|
name = "quinn-proto"
|
||||||
version = "0.11.17"
|
version = "0.11.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"
|
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -8554,9 +8550,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redis"
|
name = "redis"
|
||||||
version = "1.6.0"
|
version = "1.5.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"arcstr",
|
"arcstr",
|
||||||
@@ -8568,7 +8564,7 @@ dependencies = [
|
|||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"itoa",
|
"itoa",
|
||||||
"num-bigint 0.5.1",
|
"num-bigint 0.4.8",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"rustls",
|
"rustls",
|
||||||
@@ -8869,9 +8865,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-core-next"
|
name = "rumqttc-core-next"
|
||||||
version = "0.34.0"
|
version = "0.33.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc"
|
checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-tungstenite",
|
"async-tungstenite",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
@@ -8887,18 +8883,18 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-next"
|
name = "rumqttc-next"
|
||||||
version = "0.34.0"
|
version = "0.33.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "477c9bbfba8f3aecc7aad31c6de2eacb75822efaa18e7aeecb8d3d8e534fbf07"
|
checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rumqttc-v5-next",
|
"rumqttc-v5-next",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rumqttc-v5-next"
|
name = "rumqttc-v5-next"
|
||||||
version = "0.34.0"
|
version = "0.33.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf"
|
checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-tungstenite",
|
"async-tungstenite",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -8921,9 +8917,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "russh"
|
name = "russh"
|
||||||
version = "0.62.7"
|
version = "0.62.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
|
checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.9.2",
|
"aes 0.9.2",
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
@@ -8946,7 +8942,7 @@ dependencies = [
|
|||||||
"enum_dispatch",
|
"enum_dispatch",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures",
|
"futures",
|
||||||
"generic-array 1.4.5",
|
"generic-array 1.4.4",
|
||||||
"getrandom 0.4.3",
|
"getrandom 0.4.3",
|
||||||
"ghash",
|
"ghash",
|
||||||
"hex-literal",
|
"hex-literal",
|
||||||
@@ -9204,7 +9200,6 @@ dependencies = [
|
|||||||
"serial_test",
|
"serial_test",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"shadow-rs",
|
"shadow-rs",
|
||||||
"snap",
|
|
||||||
"socket2",
|
"socket2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
@@ -9281,7 +9276,6 @@ dependencies = [
|
|||||||
"s3s",
|
"s3s",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"smallvec",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tonic",
|
"tonic",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -9492,7 +9486,7 @@ dependencies = [
|
|||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rayon",
|
"rayon",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"spin 0.12.3",
|
"spin 0.12.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -9827,19 +9821,14 @@ name = "rustfs-madmin"
|
|||||||
version = "1.0.0-rc.2"
|
version = "1.0.0-rc.2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hotpath",
|
"hotpath",
|
||||||
"http 1.5.0",
|
|
||||||
"humantime",
|
"humantime",
|
||||||
"hyper",
|
"hyper",
|
||||||
"jiff",
|
"jiff",
|
||||||
"reqwest",
|
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rustfs-signer",
|
|
||||||
"s3s",
|
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
"time",
|
"time",
|
||||||
"tokio",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -10253,7 +10242,6 @@ dependencies = [
|
|||||||
"rustfs-ecstore",
|
"rustfs-ecstore",
|
||||||
"rustfs-filemeta",
|
"rustfs-filemeta",
|
||||||
"rustfs-lock",
|
"rustfs-lock",
|
||||||
"rustfs-s3-types",
|
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
"s3s",
|
"s3s",
|
||||||
@@ -10834,7 +10822,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"base16ct 0.2.0",
|
"base16ct 0.2.0",
|
||||||
"der 0.7.10",
|
"der 0.7.10",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"pkcs8 0.10.2",
|
"pkcs8 0.10.2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -11398,9 +11386,9 @@ checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "spin"
|
name = "spin"
|
||||||
version = "0.12.3"
|
version = "0.12.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10"
|
checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"lock_api",
|
"lock_api",
|
||||||
]
|
]
|
||||||
@@ -11812,7 +11800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fastrand",
|
"fastrand",
|
||||||
"getrandom 0.4.3",
|
"getrandom 0.3.4",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix",
|
"rustix",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
@@ -11976,9 +11964,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinystr"
|
name = "tinystr"
|
||||||
version = "0.8.4"
|
version = "0.8.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
|
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"zerovec",
|
"zerovec",
|
||||||
@@ -12652,9 +12640,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "uuid"
|
name = "uuid"
|
||||||
version = "1.24.1"
|
version = "1.24.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
|
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"getrandom 0.4.3",
|
"getrandom 0.4.3",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
@@ -13169,9 +13157,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "writeable"
|
name = "writeable"
|
||||||
version = "0.6.4"
|
version = "0.6.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "x509-cert"
|
name = "x509-cert"
|
||||||
@@ -13351,9 +13339,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerotrie"
|
name = "zerotrie"
|
||||||
version = "0.2.5"
|
version = "0.2.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
|
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"displaydoc",
|
"displaydoc",
|
||||||
"yoke",
|
"yoke",
|
||||||
@@ -13362,9 +13350,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerovec"
|
name = "zerovec"
|
||||||
version = "0.11.7"
|
version = "0.11.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
|
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"yoke",
|
"yoke",
|
||||||
"zerofrom",
|
"zerofrom",
|
||||||
@@ -13373,13 +13361,13 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerovec-derive"
|
name = "zerovec-derive"
|
||||||
version = "0.11.5"
|
version = "0.11.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
|
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.3",
|
"syn 2.0.119",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+9
-9
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
|||||||
tower-http = { version = "0.7.0" }
|
tower-http = { version = "0.7.0" }
|
||||||
|
|
||||||
# Serialization and Data Formats
|
# Serialization and Data Formats
|
||||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
apache-avro = "0.22.0"
|
||||||
bytes = { version = "1.12.1" }
|
bytes = { version = "1.12.1" }
|
||||||
bytesize = "2.7.0"
|
bytesize = "2.7.0"
|
||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
@@ -228,9 +228,9 @@ atoi = "3.1.0"
|
|||||||
atomic_enum = "0.3.0"
|
atomic_enum = "0.3.0"
|
||||||
aws-config = { version = "1.10.1" }
|
aws-config = { version = "1.10.1" }
|
||||||
aws-credential-types = { version = "1.3.0" }
|
aws-credential-types = { version = "1.3.0" }
|
||||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||||
aws-smithy-types = { version = "1.6.2" }
|
aws-smithy-types = { version = "1.6.2" }
|
||||||
@@ -284,8 +284,8 @@ rayon = "1.12.0"
|
|||||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||||
reed-solomon-simd = "3.1.0"
|
reed-solomon-simd = "3.1.0"
|
||||||
regex = { version = "1.13.1" }
|
regex = { version = "1.13.1" }
|
||||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||||
redis = { version = "1.6.0" }
|
redis = { version = "1.5.0" }
|
||||||
rustify = { version = "0.7", default-features = false }
|
rustify = { version = "0.7", default-features = false }
|
||||||
rustix = { version = "1.1.4" }
|
rustix = { version = "1.1.4" }
|
||||||
rust-embed = { version = "8.12.0" }
|
rust-embed = { version = "8.12.0" }
|
||||||
@@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" }
|
|||||||
transform-stream = "0.3.1"
|
transform-stream = "0.3.1"
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
urlencoding = "2.1.3"
|
urlencoding = "2.1.3"
|
||||||
uuid = { version = "1.24.1" }
|
uuid = { version = "1.24.0" }
|
||||||
vaultrs = { version = "0.8.0" }
|
vaultrs = { version = "0.8.0" }
|
||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
walkdir = "2.5.0"
|
walkdir = "2.5.0"
|
||||||
@@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" }
|
|||||||
unftp-core = "0.1.0"
|
unftp-core = "0.1.0"
|
||||||
suppaftp = { version = "10.0.1" }
|
suppaftp = { version = "10.0.1" }
|
||||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||||
russh = { version = "0.62.7" }
|
russh = { version = "0.62.6" }
|
||||||
russh-sftp = "2.4.0"
|
russh-sftp = "2.4.0"
|
||||||
|
|
||||||
# WebDAV
|
# WebDAV
|
||||||
@@ -350,7 +350,7 @@ dav-server = "0.11.0"
|
|||||||
# Performance Analysis and Memory Profiling
|
# Performance Analysis and Memory Profiling
|
||||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||||
hotpath = { version = "0.23.3", default-features = false }
|
hotpath = { version = "0.23.2", default-features = false }
|
||||||
# Snapshot testing for output format regression detection
|
# Snapshot testing for output format regression detection
|
||||||
insta = { version = "1.48" }
|
insta = { version = "1.48" }
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ mak = "mak"
|
|||||||
gae = "gae"
|
gae = "gae"
|
||||||
GAE = "GAE"
|
GAE = "GAE"
|
||||||
thr = "thr"
|
thr = "thr"
|
||||||
mis = "mis"
|
|
||||||
# s3-tests original test names (cannot be changed)
|
# s3-tests original test names (cannot be changed)
|
||||||
nonexisted = "nonexisted"
|
nonexisted = "nonexisted"
|
||||||
consts = "consts"
|
consts = "consts"
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
use base64_simd::STANDARD;
|
use base64_simd::STANDARD;
|
||||||
|
|
||||||
|
|||||||
@@ -38,10 +38,7 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
|||||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||||
|
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||||
|
|
||||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||||
|
|||||||
@@ -476,19 +476,13 @@ impl Checksum for Xxhash64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct Md5 {
|
struct Md5 {
|
||||||
hasher: md5::Md5,
|
hasher: md5::Md5,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
impl Md5 {
|
impl Md5 {
|
||||||
fn update(&mut self, bytes: &[u8]) {
|
fn update(&mut self, bytes: &[u8]) {
|
||||||
use md5::Digest;
|
use md5::Digest;
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ chrono = { workspace = true, features = ["serde"] }
|
|||||||
jiff = { workspace = true, features = ["serde"] }
|
jiff = { workspace = true, features = ["serde"] }
|
||||||
metrics = { workspace = true }
|
metrics = { workspace = true }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
smallvec = { workspace = true }
|
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
s3s = { workspace = true, features = ["minio"] }
|
s3s = { workspace = true, features = ["minio"] }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
|||||||
@@ -287,9 +287,6 @@ pub enum HealRequestSource {
|
|||||||
Scanner,
|
Scanner,
|
||||||
AutoHeal,
|
AutoHeal,
|
||||||
ReadRepair,
|
ReadRepair,
|
||||||
/// Mission Repair Feed: intents delivered by error paths and replayed
|
|
||||||
/// from the durable MRF journal.
|
|
||||||
Mrf,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HealRequestSource {
|
impl HealRequestSource {
|
||||||
@@ -300,7 +297,6 @@ impl HealRequestSource {
|
|||||||
Self::Scanner => "scanner",
|
Self::Scanner => "scanner",
|
||||||
Self::AutoHeal => "auto_heal",
|
Self::AutoHeal => "auto_heal",
|
||||||
Self::ReadRepair => "read_repair",
|
Self::ReadRepair => "read_repair",
|
||||||
Self::Mrf => "mrf",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,10 +17,8 @@ pub mod globals;
|
|||||||
pub mod heal_channel;
|
pub mod heal_channel;
|
||||||
pub mod last_minute;
|
pub mod last_minute;
|
||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
pub mod mrf_channel;
|
|
||||||
mod readiness;
|
mod readiness;
|
||||||
pub mod table_catalog;
|
pub mod table_catalog;
|
||||||
pub mod trace_bus;
|
|
||||||
|
|
||||||
pub use globals::*;
|
pub use globals::*;
|
||||||
pub use readiness::{GlobalReadiness, SystemStage};
|
pub use readiness::{GlobalReadiness, SystemStage};
|
||||||
|
|||||||
@@ -1,203 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//! Mission Repair Feed (MRF) intent channel.
|
|
||||||
//!
|
|
||||||
//! Producers on error paths (read decode failure, scanner metadata
|
|
||||||
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
|
||||||
//! heal crate through a global bounded channel. Delivery is strictly
|
|
||||||
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
|
||||||
//! (counting it) when the channel is full or uninitialized — losing one heal
|
|
||||||
//! hint is always preferred over stalling an IO path. Durable replay of
|
|
||||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
|
||||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
|
||||||
|
|
||||||
use std::sync::{
|
|
||||||
Arc, OnceLock,
|
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
};
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
|
||||||
/// dropping (and counting) intents, never by blocking the producer.
|
|
||||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
|
||||||
|
|
||||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
|
||||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
|
||||||
/// PartialWrite -> Normal).
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
||||||
pub enum MrfKind {
|
|
||||||
/// Erasure decode failed while serving a read (read path).
|
|
||||||
DecodeFailure,
|
|
||||||
/// Scanner classified object metadata as corrupt.
|
|
||||||
MetadataCorruption,
|
|
||||||
/// A write left the object with fewer committed shards than the set size.
|
|
||||||
PartialWrite,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MrfKind {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
MrfKind::DecodeFailure => "decode-failure",
|
|
||||||
MrfKind::MetadataCorruption => "metadata-corruption",
|
|
||||||
MrfKind::PartialWrite => "partial-write",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
|
||||||
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
|
||||||
/// intent never re-allocates the strings.
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub struct MrfIntent {
|
|
||||||
pub bucket: Arc<str>,
|
|
||||||
pub object: Arc<str>,
|
|
||||||
/// Version the intent targets, as raw UUID bytes.
|
|
||||||
pub version_id: Option<[u8; 16]>,
|
|
||||||
pub kind: MrfKind,
|
|
||||||
pub enqueued_at_ms: u64,
|
|
||||||
/// Times this intent has already been offered to the heal manager.
|
|
||||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
|
||||||
pub attempts: u8,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Consumer-side retry ceiling before an intent is given up on.
|
|
||||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
|
||||||
|
|
||||||
impl MrfIntent {
|
|
||||||
/// Rough in-memory footprint used by the queue's byte budget.
|
|
||||||
pub fn estimated_bytes(&self) -> usize {
|
|
||||||
// Struct + strings + version bytes; buckets and objects are usually
|
|
||||||
// far below this bound, so rounding up keeps the budget conservative.
|
|
||||||
64 + self.bucket.len() + self.object.len()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
|
||||||
|
|
||||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
|
||||||
/// this before touching the channel so the disabled path stays allocation- and
|
|
||||||
/// sync-free.
|
|
||||||
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
|
||||||
|
|
||||||
/// Override delivery (used at heal-runtime startup from configuration).
|
|
||||||
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
|
||||||
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether producers currently deliver intents.
|
|
||||||
pub fn mrf_delivery_enabled() -> bool {
|
|
||||||
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create the global MRF channel and return the consumer half. Fails if the
|
|
||||||
/// channel is already initialized (the heal runtime is a singleton).
|
|
||||||
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
|
||||||
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
|
||||||
GLOBAL_MRF_SENDER
|
|
||||||
.set(sender)
|
|
||||||
.map_err(|_| "MRF channel sender already initialized")?;
|
|
||||||
Ok(receiver)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Best-effort, non-blocking intent delivery from an error path.
|
|
||||||
///
|
|
||||||
/// Returns `true` when the intent was accepted into the channel. `false`
|
|
||||||
/// means the intent was dropped (feature disabled, channel not yet
|
|
||||||
/// initialized, or channel full) — callers must not retry or await; the
|
|
||||||
/// existing read-repair / scanner heal paths remain the safety net.
|
|
||||||
///
|
|
||||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
|
||||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
|
||||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
|
||||||
if !mrf_delivery_enabled() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
let intent = MrfIntent {
|
|
||||||
bucket: Arc::from(bucket),
|
|
||||||
object: Arc::from(object),
|
|
||||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
|
||||||
kind,
|
|
||||||
enqueued_at_ms: unix_now_ms(),
|
|
||||||
attempts: 0,
|
|
||||||
};
|
|
||||||
sender.try_send(intent).is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unix_now_ms() -> u64 {
|
|
||||||
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
|
||||||
// failure would be a bug rather than something to handle here.
|
|
||||||
std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_millis() as u64)
|
|
||||||
.unwrap_or(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn intents_estimate_is_conservative() {
|
|
||||||
let intent = MrfIntent {
|
|
||||||
bucket: Arc::from("bucket"),
|
|
||||||
object: Arc::from("object"),
|
|
||||||
version_id: Some([0u8; 16]),
|
|
||||||
kind: MrfKind::DecodeFailure,
|
|
||||||
enqueued_at_ms: 0,
|
|
||||||
attempts: 0,
|
|
||||||
};
|
|
||||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn try_send_delivers_and_respects_capacity() {
|
|
||||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
|
||||||
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
|
||||||
|
|
||||||
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
|
||||||
let intent = receiver.recv().await.expect("intent should arrive");
|
|
||||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
|
||||||
assert_eq!(intent.bucket.as_ref(), "b");
|
|
||||||
|
|
||||||
// Disable delivery: producers become no-ops.
|
|
||||||
set_mrf_delivery_enabled(false);
|
|
||||||
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
|
||||||
set_mrf_delivery_enabled(true);
|
|
||||||
|
|
||||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
|
||||||
// never blocking.
|
|
||||||
let mut accepted = 0;
|
|
||||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
|
||||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
|
||||||
accepted += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn try_send_without_channel_is_false() {
|
|
||||||
// This test may run after the tokio test above in the same process;
|
|
||||||
// the singleton semantics make a clean "uninitialized" case hard, so
|
|
||||||
// assert the flag-off behavior only.
|
|
||||||
set_mrf_delivery_enabled(false);
|
|
||||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
|
||||||
set_mrf_delivery_enabled(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,333 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
use smallvec::SmallVec;
|
|
||||||
use std::{
|
|
||||||
sync::{
|
|
||||||
Arc, OnceLock,
|
|
||||||
atomic::{AtomicUsize, Ordering},
|
|
||||||
},
|
|
||||||
time::{Duration, SystemTime},
|
|
||||||
};
|
|
||||||
use tokio::sync::broadcast;
|
|
||||||
|
|
||||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
|
||||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
|
||||||
|
|
||||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum TraceKind {
|
|
||||||
Heal,
|
|
||||||
Scanner,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceKind {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Heal => "heal",
|
|
||||||
Self::Scanner => "scanner",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum TraceFunc {
|
|
||||||
HealTask,
|
|
||||||
HealBucket,
|
|
||||||
HealObject,
|
|
||||||
HealCheckAbandonedParts,
|
|
||||||
HealErasureSetPage,
|
|
||||||
ScannerFolder,
|
|
||||||
ScannerIlmAction,
|
|
||||||
ScannerHealCandidate,
|
|
||||||
Dropped,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceFunc {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::HealTask => "heal.Task",
|
|
||||||
Self::HealBucket => "heal.Bucket",
|
|
||||||
Self::HealObject => "heal.Object",
|
|
||||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
|
||||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
|
||||||
Self::ScannerFolder => "scanner.Folder",
|
|
||||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
|
||||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
|
||||||
Self::Dropped => "trace.Dropped",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum TraceVal {
|
|
||||||
Bool(bool),
|
|
||||||
U64(u64),
|
|
||||||
I64(i64),
|
|
||||||
Str(Arc<str>),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<bool> for TraceVal {
|
|
||||||
fn from(value: bool) -> Self {
|
|
||||||
Self::Bool(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<u64> for TraceVal {
|
|
||||||
fn from(value: u64) -> Self {
|
|
||||||
Self::U64(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<i64> for TraceVal {
|
|
||||||
fn from(value: i64) -> Self {
|
|
||||||
Self::I64(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<&str> for TraceVal {
|
|
||||||
fn from(value: &str) -> Self {
|
|
||||||
Self::Str(Arc::from(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<String> for TraceVal {
|
|
||||||
fn from(value: String) -> Self {
|
|
||||||
Self::Str(Arc::from(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct TraceAttr {
|
|
||||||
pub key: &'static str,
|
|
||||||
pub value: TraceVal,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct TraceEvent {
|
|
||||||
pub kind: TraceKind,
|
|
||||||
pub func: TraceFunc,
|
|
||||||
pub time: SystemTime,
|
|
||||||
pub bucket: Option<Arc<str>>,
|
|
||||||
pub object: Option<Arc<str>>,
|
|
||||||
pub duration: Duration,
|
|
||||||
pub bytes: u64,
|
|
||||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceEvent {
|
|
||||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
|
||||||
Self {
|
|
||||||
kind,
|
|
||||||
func,
|
|
||||||
time: SystemTime::now(),
|
|
||||||
bucket: None,
|
|
||||||
object: None,
|
|
||||||
duration: Duration::ZERO,
|
|
||||||
bytes: 0,
|
|
||||||
attrs: SmallVec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
|
||||||
self.bucket = Some(bucket.into());
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
|
||||||
self.object = Some(object.into());
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
|
||||||
self.duration = duration;
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
|
||||||
self.bytes = bytes;
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
|
||||||
self.attrs.push(TraceAttr {
|
|
||||||
key,
|
|
||||||
value: value.into(),
|
|
||||||
});
|
|
||||||
self
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct TraceBus {
|
|
||||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
|
||||||
subscriber_count: Arc<AtomicUsize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceBus {
|
|
||||||
pub fn new(capacity: usize) -> Self {
|
|
||||||
let capacity = capacity.max(1);
|
|
||||||
let (sender, _receiver) = broadcast::channel(capacity);
|
|
||||||
Self {
|
|
||||||
sender,
|
|
||||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscriber_count(&self) -> usize {
|
|
||||||
self.subscriber_count.load(Ordering::Acquire)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscribe(&self) -> TraceSubscription {
|
|
||||||
let receiver = self.sender.subscribe();
|
|
||||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
|
||||||
TraceSubscription {
|
|
||||||
receiver,
|
|
||||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
|
||||||
if self.subscriber_count() == 0 {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
self.sender.send(Arc::new(build())).is_ok()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for TraceBus {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct TraceSubscription {
|
|
||||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
|
||||||
subscriber_count: Arc<AtomicUsize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceSubscription {
|
|
||||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
|
||||||
self.receiver.recv().await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
|
||||||
self.receiver.try_recv()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for TraceSubscription {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn global_trace_bus() -> &'static TraceBus {
|
|
||||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
|
||||||
global_trace_bus().subscribe()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
|
||||||
global_trace_bus().emit(build)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn trace_subscriber_count() -> usize {
|
|
||||||
global_trace_bus().subscriber_count()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn trace_emit_skips_builder_without_subscribers() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let built = AtomicUsize::new(0);
|
|
||||||
|
|
||||||
let sent = bus.emit(|| {
|
|
||||||
built.fetch_add(1, Ordering::Relaxed);
|
|
||||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
|
||||||
});
|
|
||||||
|
|
||||||
assert!(!sent);
|
|
||||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn trace_subscriber_receives_event() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let mut subscription = bus.subscribe();
|
|
||||||
|
|
||||||
assert!(bus.emit(|| {
|
|
||||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
|
||||||
.with_bucket("bucket")
|
|
||||||
.with_object("object")
|
|
||||||
.with_duration(Duration::from_millis(7))
|
|
||||||
.with_bytes(11)
|
|
||||||
.with_attr("dry", true)
|
|
||||||
}));
|
|
||||||
|
|
||||||
let event = subscription
|
|
||||||
.recv()
|
|
||||||
.await
|
|
||||||
.expect("subscriber should receive emitted trace event");
|
|
||||||
|
|
||||||
assert_eq!(event.kind, TraceKind::Heal);
|
|
||||||
assert_eq!(event.func, TraceFunc::HealObject);
|
|
||||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
|
||||||
assert_eq!(event.object.as_deref(), Some("object"));
|
|
||||||
assert_eq!(event.duration, Duration::from_millis(7));
|
|
||||||
assert_eq!(event.bytes, 11);
|
|
||||||
assert_eq!(
|
|
||||||
event.attrs.as_slice(),
|
|
||||||
&[TraceAttr {
|
|
||||||
key: "dry",
|
|
||||||
value: TraceVal::Bool(true)
|
|
||||||
}]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn trace_subscription_drop_decrements_count() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let subscription = bus.subscribe();
|
|
||||||
|
|
||||||
assert_eq!(bus.subscriber_count(), 1);
|
|
||||||
drop(subscription);
|
|
||||||
assert_eq!(bus.subscriber_count(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
|
||||||
let bus = TraceBus::new(2);
|
|
||||||
let mut subscription = bus.subscribe();
|
|
||||||
|
|
||||||
for index in 0_u64..4 {
|
|
||||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
|
||||||
}
|
|
||||||
|
|
||||||
let err = subscription
|
|
||||||
.recv()
|
|
||||||
.await
|
|
||||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
|
||||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,8 +14,9 @@
|
|||||||
|
|
||||||
//! Shared backpressure policy type.
|
//! Shared backpressure policy type.
|
||||||
//!
|
//!
|
||||||
//! This module only carries the watermark policy; the admission primitive it
|
//! The runtime backpressure implementation (byte-watermark pipes and
|
||||||
//! projects into lives in `rustfs-io-core`.
|
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
||||||
|
//! carries the watermark policy type that implementation shares.
|
||||||
|
|
||||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||||
|
|
||||||
|
|||||||
@@ -177,31 +177,3 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
|||||||
|
|
||||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||||
|
|
||||||
/// Environment variable that toggles the MRF (mission repair feed) intent
|
|
||||||
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
|
||||||
/// unconsumed intents are replayed from the durable journal after a restart.
|
|
||||||
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
|
||||||
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF journal byte budget. The journal is
|
|
||||||
/// compacted once its on-disk size crosses this bound.
|
|
||||||
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF journal replay batch size (intents per
|
|
||||||
/// replay push round).
|
|
||||||
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
|
||||||
|
|
||||||
/// Default behavior keeps the MRF intent pipeline enabled.
|
|
||||||
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
|
||||||
|
|
||||||
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
|
||||||
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
|
||||||
|
|
||||||
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
|
||||||
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
|
||||||
|
|
||||||
/// Default MRF replay batch size.
|
|
||||||
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
|
||||||
|
|||||||
@@ -234,31 +234,6 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
|||||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||||
|
|
||||||
/// Enable foreground PutObject request admission.
|
|
||||||
///
|
|
||||||
/// This is an experimental, default-off foreground write backpressure gate for
|
|
||||||
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
|
||||||
/// path and only the existing request counters are updated.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
|
||||||
|
|
||||||
/// Maximum foreground PutObject requests admitted concurrently per process.
|
|
||||||
///
|
|
||||||
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
|
||||||
/// A value of `0` disables the gate even when the enable flag is present, so a
|
|
||||||
/// partially configured rollout cannot reject every PUT.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
|
||||||
|
|
||||||
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
|
||||||
///
|
|
||||||
/// Once this timeout expires the request fails before body ingest/storage
|
|
||||||
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
|
||||||
|
|
||||||
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
|
||||||
|
|
||||||
/// Environment variable for minimum GetObject timeout in seconds.
|
/// Environment variable for minimum GetObject timeout in seconds.
|
||||||
///
|
///
|
||||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||||
|
|||||||
@@ -870,157 +870,6 @@ pub struct DataUsageCacheInfo {
|
|||||||
pub snapshot_complete: bool,
|
pub snapshot_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
|
||||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
|
||||||
/// scanner's writer-side cache has the same map type).
|
|
||||||
///
|
|
||||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
|
||||||
/// names come straight off the child keys — no reverse mapping exists or is
|
|
||||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
|
||||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
|
||||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
|
||||||
/// was cut (largest first).
|
|
||||||
pub fn prefix_usage_in_cache(
|
|
||||||
cache: &HashMap<String, DataUsageEntry>,
|
|
||||||
bucket: &str,
|
|
||||||
prefix: &str,
|
|
||||||
max_entries: usize,
|
|
||||||
) -> Option<PrefixUsageQuery> {
|
|
||||||
let prefix = prefix.trim_matches('/');
|
|
||||||
let root = if prefix.is_empty() {
|
|
||||||
bucket.to_string()
|
|
||||||
} else {
|
|
||||||
format!("{bucket}/{prefix}")
|
|
||||||
};
|
|
||||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
|
||||||
|
|
||||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
|
||||||
|
|
||||||
let child_prefix = format!("{root}/");
|
|
||||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
|
||||||
.children
|
|
||||||
.iter()
|
|
||||||
.filter_map(|child_key| {
|
|
||||||
let child = cache.get(child_key)?;
|
|
||||||
let child_flat = flatten_entry(cache, child, 1)?;
|
|
||||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
|
||||||
// slash marks a directory object and is display-only here.
|
|
||||||
let name = child_key
|
|
||||||
.strip_prefix(child_prefix.as_str())
|
|
||||||
.unwrap_or(child_key.as_str())
|
|
||||||
.trim_end_matches('/')
|
|
||||||
.to_string();
|
|
||||||
Some(PrefixUsageEntry {
|
|
||||||
prefix: name,
|
|
||||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
sub_prefixes.sort_by(|left, right| {
|
|
||||||
right
|
|
||||||
.usage
|
|
||||||
.size
|
|
||||||
.cmp(&left.usage.size)
|
|
||||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
|
||||||
});
|
|
||||||
let truncated = sub_prefixes.len() > max_entries;
|
|
||||||
sub_prefixes.truncate(max_entries);
|
|
||||||
|
|
||||||
Some(PrefixUsageQuery {
|
|
||||||
usage,
|
|
||||||
compacted: entry.compacted,
|
|
||||||
truncated,
|
|
||||||
sub_prefixes,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
|
||||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
|
||||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
|
||||||
|
|
||||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
|
||||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
|
||||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
|
||||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
|
||||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
|
||||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let mut flattened = DataUsageEntry::default();
|
|
||||||
if !flattened.checked_merge(root) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
flattened.compacted = root.compacted;
|
|
||||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
|
||||||
// child edge pointing back at the root's own key is still terminated by
|
|
||||||
// the visited set on first encounter.
|
|
||||||
let mut visited: HashSet<&str> = HashSet::new();
|
|
||||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
|
||||||
while let Some((key, child_depth)) = pending.pop() {
|
|
||||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let entry = cache.get(key)?;
|
|
||||||
if !flattened.checked_merge(entry) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
|
||||||
}
|
|
||||||
flattened.children.clear();
|
|
||||||
Some(flattened)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Flattened counters of one prefix subtree, as returned by
|
|
||||||
/// [`DataUsageCache::prefix_usage`].
|
|
||||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PrefixUsageSummary {
|
|
||||||
pub size: u64,
|
|
||||||
pub objects: u64,
|
|
||||||
pub versions: u64,
|
|
||||||
pub delete_markers: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PrefixUsageSummary {
|
|
||||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
|
||||||
Self {
|
|
||||||
size: entry.size as u64,
|
|
||||||
objects: entry.objects as u64,
|
|
||||||
versions: entry.versions as u64,
|
|
||||||
delete_markers: entry.delete_markers as u64,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Add another set's counters into this one (entries are partitioned by
|
|
||||||
/// set, so per-set results sum).
|
|
||||||
pub fn merge(&mut self, other: &Self) {
|
|
||||||
self.size = self.size.saturating_add(other.size);
|
|
||||||
self.objects = self.objects.saturating_add(other.objects);
|
|
||||||
self.versions = self.versions.saturating_add(other.versions);
|
|
||||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
|
||||||
pub struct PrefixUsageEntry {
|
|
||||||
pub prefix: String,
|
|
||||||
pub usage: PrefixUsageSummary,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Result of [`DataUsageCache::prefix_usage`].
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PrefixUsageQuery {
|
|
||||||
pub usage: PrefixUsageSummary,
|
|
||||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
|
||||||
/// but no sub-prefix breakdown exists on disk.
|
|
||||||
pub compacted: bool,
|
|
||||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
|
||||||
pub truncated: bool,
|
|
||||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||||
///
|
///
|
||||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||||
@@ -1148,21 +997,6 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
|
||||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
|
||||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
|
||||||
/// full counters instead of first-level sizes only).
|
|
||||||
///
|
|
||||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
|
||||||
/// names come straight off the child keys — no reverse mapping exists or
|
|
||||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
|
||||||
/// which the `compacted` flag reports so callers can say why the
|
|
||||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
|
||||||
/// `max_entries` and was cut (largest first).
|
|
||||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
|
||||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn force_compact(&mut self, limit: usize) {
|
pub fn force_compact(&mut self, limit: usize) {
|
||||||
if self.cache.len() < limit {
|
if self.cache.len() < limit {
|
||||||
return;
|
return;
|
||||||
@@ -2064,126 +1898,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
|
||||||
/// distinct counters so aggregation is observable.
|
|
||||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
|
||||||
let mut cache = DataUsageCache::default();
|
|
||||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
|
||||||
cache.replace(
|
|
||||||
path,
|
|
||||||
parent,
|
|
||||||
DataUsageEntry {
|
|
||||||
size,
|
|
||||||
objects,
|
|
||||||
versions,
|
|
||||||
delete_markers,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
};
|
|
||||||
insert("bucket", "", 0, 0, 0, 0);
|
|
||||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
|
||||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
|
||||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
|
||||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
|
||||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
|
||||||
cache
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
|
|
||||||
let root = cache
|
|
||||||
.prefix_usage("bucket", "", 100)
|
|
||||||
.expect("root query must find the bucket entry");
|
|
||||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
|
||||||
assert_eq!(root.usage.objects, 5);
|
|
||||||
assert_eq!(root.usage.versions, 7);
|
|
||||||
assert_eq!(root.usage.delete_markers, 2);
|
|
||||||
assert!(!root.compacted);
|
|
||||||
assert!(!root.truncated);
|
|
||||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
|
||||||
// each flattened to its own subtree total.
|
|
||||||
let names: Vec<(&str, u64)> = root
|
|
||||||
.sub_prefixes
|
|
||||||
.iter()
|
|
||||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
|
||||||
.collect();
|
|
||||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
|
|
||||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
|
||||||
assert_eq!(b.usage.size, 240);
|
|
||||||
assert_eq!(b.usage.versions, 5);
|
|
||||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
|
||||||
assert_eq!(names, vec!["c", "d"]);
|
|
||||||
|
|
||||||
// Prefix slashes are normalized away.
|
|
||||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
|
||||||
assert_eq!(slashed.usage.size, 240);
|
|
||||||
|
|
||||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
|
||||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_reports_and_respects_truncation() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
|
||||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
|
||||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
|
||||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_marks_compacted_entries() {
|
|
||||||
let mut cache = DataUsageCache::default();
|
|
||||||
cache.replace(
|
|
||||||
"bucket",
|
|
||||||
"",
|
|
||||||
DataUsageEntry {
|
|
||||||
size: 999,
|
|
||||||
objects: 9,
|
|
||||||
compacted: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
|
||||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
|
||||||
assert_eq!(compacted.usage.size, 999);
|
|
||||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
|
||||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
|
||||||
// whole query, not unbounded recursion.
|
|
||||||
let mut cache = prefix_usage_fixture_cache();
|
|
||||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
|
||||||
entry.children.insert("bucket/b".to_string());
|
|
||||||
}
|
|
||||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
|
||||||
// The unaffected sibling still answers.
|
|
||||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
|
||||||
|
|
||||||
// A child key with no entry (dangling link) is rejected rather than
|
|
||||||
// silently dropped: half a tree would under-report usage.
|
|
||||||
let mut dangling = prefix_usage_fixture_cache();
|
|
||||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
|
||||||
entry.children.insert("bucket/b/ghost".to_string());
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
|
||||||
"a dangling child link must be rejected"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hash_path_uses_portable_slash_semantics() {
|
fn hash_path_uses_portable_slash_semantics() {
|
||||||
for (input, expected) in [
|
for (input, expected) in [
|
||||||
|
|||||||
@@ -32,7 +32,6 @@ use rustfs_signer::sign_v4;
|
|||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::fs as stdfs;
|
use std::fs as stdfs;
|
||||||
use std::io::ErrorKind;
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
use std::sync::Once;
|
use std::sync::Once;
|
||||||
@@ -52,11 +51,6 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
|||||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||||
const TEST_PORT_MIN: u16 = 20_000;
|
|
||||||
const TEST_PORT_RANGE: u16 = 40_000;
|
|
||||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
|
||||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
|
||||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
|
||||||
|
|
||||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||||
@@ -73,64 +67,6 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
|||||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||||
}
|
}
|
||||||
|
|
||||||
struct PortAllocatorGuard;
|
|
||||||
|
|
||||||
impl PortAllocatorGuard {
|
|
||||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
loop {
|
|
||||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
|
||||||
Ok(()) => return Ok(Self),
|
|
||||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
|
||||||
remove_stale_port_allocator_lock();
|
|
||||||
sleep(Duration::from_millis(10)).await;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(err.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for PortAllocatorGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn advance_test_port(port: u16) -> u16 {
|
|
||||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
|
||||||
TEST_PORT_MIN + offset
|
|
||||||
}
|
|
||||||
|
|
||||||
fn seeded_test_port() -> u16 {
|
|
||||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
|
||||||
TEST_PORT_MIN + offset
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_next_test_port() -> u16 {
|
|
||||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
|
||||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
|
||||||
.unwrap_or_else(seeded_test_port)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn remove_stale_port_allocator_lock() {
|
|
||||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let Ok(modified) = metadata.modified() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
|
||||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn capture_command_logs(
|
pub(crate) fn capture_command_logs(
|
||||||
command: &mut Command,
|
command: &mut Command,
|
||||||
log_path: Option<&str>,
|
log_path: Option<&str>,
|
||||||
@@ -572,21 +508,10 @@ impl RustFSTestEnvironment {
|
|||||||
/// Find an available port for the test
|
/// Find an available port for the test
|
||||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
let _guard = PortAllocatorGuard::acquire().await?;
|
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||||
let mut next_port = read_next_test_port();
|
let port = listener.local_addr()?.port();
|
||||||
|
drop(listener);
|
||||||
for _ in 0..TEST_PORT_RANGE {
|
Ok(port)
|
||||||
let port = next_port;
|
|
||||||
next_port = advance_test_port(next_port);
|
|
||||||
write_next_test_port(next_port)?;
|
|
||||||
|
|
||||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
|
||||||
drop(listener);
|
|
||||||
return Ok(port);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Err("no available E2E test port found".into())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kill any existing RustFS processes
|
/// Kill any existing RustFS processes
|
||||||
|
|||||||
@@ -76,18 +76,6 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
|||||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-tagging-timestamp",
|
|
||||||
"x-minio-source-replication-tagging-timestamp",
|
|
||||||
];
|
|
||||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-retention-timestamp",
|
|
||||||
"x-minio-source-replication-retention-timestamp",
|
|
||||||
];
|
|
||||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-legalhold-timestamp",
|
|
||||||
"x-minio-source-replication-legalhold-timestamp",
|
|
||||||
];
|
|
||||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||||
|
|
||||||
@@ -130,25 +118,6 @@ pub enum FaultAction {
|
|||||||
WrongEtag,
|
WrongEtag,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
|
||||||
/// sender-side tests can assert what a real target would receive.
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
||||||
pub struct ReplicationTimestampHeaders {
|
|
||||||
pub tagging: Option<String>,
|
|
||||||
pub retention: Option<String>,
|
|
||||||
pub legalhold: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReplicationTimestampHeaders {
|
|
||||||
fn from_headers(headers: &HeaderMap) -> Self {
|
|
||||||
Self {
|
|
||||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Credential-free request metadata retained for deterministic assertions.
|
/// Credential-free request metadata retained for deterministic assertions.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct RequestRecord {
|
pub struct RequestRecord {
|
||||||
@@ -162,7 +131,6 @@ pub struct RequestRecord {
|
|||||||
pub part_number: Option<i32>,
|
pub part_number: Option<i32>,
|
||||||
pub content_length: Option<u64>,
|
pub content_length: Option<u64>,
|
||||||
pub consumed_bytes: Option<usize>,
|
pub consumed_bytes: Option<usize>,
|
||||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
|
||||||
pub fault: Option<FaultAction>,
|
pub fault: Option<FaultAction>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -568,15 +536,7 @@ impl S3Access for FaultAccess {
|
|||||||
.get(CONTENT_LENGTH)
|
.get(CONTENT_LENGTH)
|
||||||
.and_then(|value| value.to_str().ok())
|
.and_then(|value| value.to_str().ok())
|
||||||
.and_then(|value| value.parse().ok());
|
.and_then(|value| value.parse().ok());
|
||||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||||
let fault = record_request(
|
|
||||||
&self.control,
|
|
||||||
operation,
|
|
||||||
context.method().clone(),
|
|
||||||
parsed,
|
|
||||||
content_length,
|
|
||||||
replication_timestamps,
|
|
||||||
);
|
|
||||||
if let Some(RequestFault {
|
if let Some(RequestFault {
|
||||||
action: FaultAction::Status(status),
|
action: FaultAction::Status(status),
|
||||||
..
|
..
|
||||||
@@ -629,7 +589,6 @@ fn record_request(
|
|||||||
method: Method,
|
method: Method,
|
||||||
parsed: ParsedRequest,
|
parsed: ParsedRequest,
|
||||||
content_length: Option<u64>,
|
content_length: Option<u64>,
|
||||||
replication_timestamps: ReplicationTimestampHeaders,
|
|
||||||
) -> Option<RequestFault> {
|
) -> Option<RequestFault> {
|
||||||
let mut state = lock(control);
|
let mut state = lock(control);
|
||||||
let action = parsed
|
let action = parsed
|
||||||
@@ -654,7 +613,6 @@ fn record_request(
|
|||||||
part_number: parsed.part_number,
|
part_number: parsed.part_number,
|
||||||
content_length,
|
content_length,
|
||||||
consumed_bytes: None,
|
consumed_bytes: None,
|
||||||
replication_timestamps,
|
|
||||||
fault: action.clone(),
|
fault: action.clone(),
|
||||||
});
|
});
|
||||||
action.map(|action| RequestFault { sequence, action })
|
action.map(|action| RequestFault { sequence, action })
|
||||||
@@ -1741,52 +1699,6 @@ mod tests {
|
|||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
|
||||||
let target = FakeS3Target::start().await?;
|
|
||||||
target.create_bucket("target-bucket");
|
|
||||||
let client = client(&target);
|
|
||||||
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket("target-bucket")
|
|
||||||
.key("plain")
|
|
||||||
.body(ByteStream::from_static(b"plain"))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket("target-bucket")
|
|
||||||
.key("stamped")
|
|
||||||
.body(ByteStream::from_static(b"stamped"))
|
|
||||||
.customize()
|
|
||||||
.map_request(move |mut request| {
|
|
||||||
let headers = request.headers_mut();
|
|
||||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
|
||||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
|
||||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
|
||||||
Ok::<_, std::convert::Infallible>(request)
|
|
||||||
})
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let requests = target.requests();
|
|
||||||
let plain = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
|
||||||
.expect("plain PUT must be journaled");
|
|
||||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
|
||||||
|
|
||||||
let stamped = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
|
||||||
.expect("stamped PUT must be journaled");
|
|
||||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
|
||||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
|
||||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! assert_sdk_error {
|
macro_rules! assert_sdk_error {
|
||||||
($error:expr, $status:expr, $code:expr) => {{
|
($error:expr, $status:expr, $code:expr) => {{
|
||||||
let error = &$error;
|
let error = &$error;
|
||||||
@@ -3073,7 +2985,6 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
Some(0),
|
Some(0),
|
||||||
ReplicationTimestampHeaders::default(),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let records = lock(&control).requests.clone();
|
let records = lock(&control).requests.clone();
|
||||||
@@ -3095,7 +3006,6 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
None,
|
None,
|
||||||
ReplicationTimestampHeaders::default(),
|
|
||||||
);
|
);
|
||||||
{
|
{
|
||||||
let bounded_records = lock(&bounded_control);
|
let bounded_records = lock(&bounded_control);
|
||||||
|
|||||||
@@ -135,8 +135,7 @@ pub mod bucket {
|
|||||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||||
pub use crate::bucket::metadata_sys::{
|
pub use crate::bucket::metadata_sys::{
|
||||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
|
||||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||||
@@ -185,18 +184,17 @@ pub mod bucket {
|
|||||||
mrf_backlog_observability_snapshot,
|
mrf_backlog_observability_snapshot,
|
||||||
};
|
};
|
||||||
pub use crate::bucket::replication::{
|
pub use crate::bucket::replication::{
|
||||||
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||||
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||||
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
|
||||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||||
@@ -373,14 +371,14 @@ pub mod error {
|
|||||||
|
|
||||||
pub mod erasure {
|
pub mod erasure {
|
||||||
pub use crate::erasure::coding::{
|
pub use crate::erasure::coding::{
|
||||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
calc_shard_size, calc_shard_size_legacy,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod event {
|
pub mod event {
|
||||||
pub use crate::event::name::EventName;
|
pub use crate::event::name::EventName;
|
||||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod global {
|
pub mod global {
|
||||||
@@ -483,7 +481,6 @@ pub mod store_list {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub mod storage {
|
pub mod storage {
|
||||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
|
||||||
pub use crate::store::HealWalkVersion;
|
pub use crate::store::HealWalkVersion;
|
||||||
pub use crate::store::{
|
pub use crate::store::{
|
||||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||||
|
|||||||
@@ -58,9 +58,7 @@ use rustfs_utils::http::{
|
|||||||
};
|
};
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
|
||||||
insert_header,
|
|
||||||
};
|
};
|
||||||
use rustls_pki_types::pem::PemObject;
|
use rustls_pki_types::pem::PemObject;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -82,6 +80,7 @@ use tracing::warn;
|
|||||||
use url::Url;
|
use url::Url;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||||
|
|
||||||
@@ -1477,12 +1476,9 @@ impl Default for AdvancedPutOptions {
|
|||||||
replication_status: ReplicationStatusType::Pending,
|
replication_status: ReplicationStatusType::Pending,
|
||||||
source_mtime: OffsetDateTime::now_utc(),
|
source_mtime: OffsetDateTime::now_utc(),
|
||||||
replication_request: false,
|
replication_request: false,
|
||||||
// UNIX_EPOCH means "never modified": header() must not emit a
|
retention_timestamp: OffsetDateTime::now_utc(),
|
||||||
// timestamp header for it, otherwise a receiver would treat an
|
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||||
// unset category as a modification made right now.
|
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||||
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
|
||||||
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
|
||||||
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
|
||||||
replication_validity_check: false,
|
replication_validity_check: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1549,8 +1545,8 @@ impl Default for PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl PutObjectOptions {
|
impl PutObjectOptions {
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn set_match_etag(&mut self, etag: &str) {
|
fn set_match_etag(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header
|
self.custom_header
|
||||||
@@ -1561,7 +1557,6 @@ impl PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn set_match_etag_except(&mut self, etag: &str) {
|
fn set_match_etag_except(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header
|
self.custom_header
|
||||||
@@ -1680,16 +1675,6 @@ impl PutObjectOptions {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (suffix, timestamp) in [
|
|
||||||
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
|
||||||
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
|
||||||
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
|
||||||
] {
|
|
||||||
if timestamp.unix_timestamp() != 0 {
|
|
||||||
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if self.internal.replication_request {
|
if self.internal.replication_request {
|
||||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||||
}
|
}
|
||||||
@@ -1697,7 +1682,6 @@ impl PutObjectOptions {
|
|||||||
header
|
header
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||||
//if self.checksum.is_set() {
|
//if self.checksum.is_set() {
|
||||||
/*if !self.trailing_header_support {
|
/*if !self.trailing_header_support {
|
||||||
@@ -2858,57 +2842,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn put_object_headers_carry_replication_timestamp_headers() {
|
|
||||||
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
|
||||||
// last-writer-wins on these headers (object-api-options.go parses them
|
|
||||||
// as RFC3339); a replica without them loses every conflict resolution.
|
|
||||||
let mut opts = PutObjectOptions::default();
|
|
||||||
opts.internal.replication_request = true;
|
|
||||||
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
|
||||||
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
|
||||||
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
|
||||||
opts.internal.tagging_timestamp = tagging;
|
|
||||||
opts.internal.retention_timestamp = retention;
|
|
||||||
opts.internal.legalhold_timestamp = legalhold;
|
|
||||||
|
|
||||||
let header = opts.header();
|
|
||||||
for (suffix, expected) in [
|
|
||||||
("source-replication-tagging-timestamp", tagging),
|
|
||||||
("source-replication-retention-timestamp", retention),
|
|
||||||
("source-replication-legalhold-timestamp", legalhold),
|
|
||||||
] {
|
|
||||||
assert_eq!(
|
|
||||||
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
|
||||||
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
|
||||||
"replication put requests must carry the {suffix} header"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn put_object_headers_omit_unset_replication_timestamps() {
|
|
||||||
// UNIX_EPOCH means "never modified on the source"; sending it would
|
|
||||||
// make the receiver treat an unset category as a fresh modification.
|
|
||||||
let mut opts = PutObjectOptions::default();
|
|
||||||
opts.internal.replication_request = true;
|
|
||||||
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
|
||||||
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
|
||||||
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
|
||||||
|
|
||||||
let header = opts.header();
|
|
||||||
for suffix in [
|
|
||||||
"source-replication-tagging-timestamp",
|
|
||||||
"source-replication-retention-timestamp",
|
|
||||||
"source-replication-legalhold-timestamp",
|
|
||||||
] {
|
|
||||||
assert!(
|
|
||||||
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
|
||||||
"unset {suffix} must not be sent to replication targets"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||||
let sys = BucketTargetSys::default();
|
let sys = BucketTargetSys::default();
|
||||||
|
|||||||
@@ -126,23 +126,11 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
|
|||||||
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
||||||
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
||||||
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
||||||
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
||||||
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub type TraceFn =
|
pub type TraceFn =
|
||||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||||
@@ -152,21 +140,9 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
|||||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||||
|
|
||||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
||||||
|
|
||||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||||
@@ -304,10 +280,6 @@ impl LifecycleSys {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||||
let bucket = oi.bucket.clone();
|
let bucket = oi.bucket.clone();
|
||||||
let name = oi.name.clone();
|
let name = oi.name.clone();
|
||||||
@@ -456,23 +428,16 @@ impl<'a> LifecycleExpiryTrace<'a> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl ExpiryStats {
|
impl ExpiryStats {
|
||||||
pub fn missed_tasks(&self) -> i64 {
|
pub fn missed_tasks(&self) -> i64 {
|
||||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn missed_free_vers_tasks(&self) -> i64 {
|
fn missed_free_vers_tasks(&self) -> i64 {
|
||||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
@@ -605,10 +570,6 @@ async fn delete_free_version_remote_object(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
||||||
oi: &ObjectInfo,
|
oi: &ObjectInfo,
|
||||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||||
@@ -1783,7 +1744,7 @@ impl TransitionState {
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
global_metrics().record_scanner_transition_failed(1);
|
global_metrics().record_scanner_transition_failed(1);
|
||||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
|
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||||
error!(
|
error!(
|
||||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
@@ -2907,10 +2868,6 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
|||||||
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||||
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||||
}
|
}
|
||||||
@@ -3395,10 +3352,6 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
||||||
if remote_delete_succeeded {
|
if remote_delete_succeeded {
|
||||||
opts.skip_decommissioned = true;
|
opts.skip_decommissioned = true;
|
||||||
@@ -4386,10 +4339,6 @@ pub async fn expire_transitioned_object(
|
|||||||
Ok(dobj)
|
Ok(dobj)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||||
let us = Uuid::new_v4().to_string();
|
let us = Uuid::new_v4().to_string();
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
@@ -4424,10 +4373,6 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||||
let tier = _tier.to_string();
|
let tier = _tier.to_string();
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
@@ -4446,10 +4391,6 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn get_transitioned_object_reader(
|
pub async fn get_transitioned_object_reader(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
@@ -5204,10 +5145,6 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
|
|||||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||||
let mut success = false;
|
let mut success = false;
|
||||||
match event.action {
|
match event.action {
|
||||||
@@ -7485,10 +7422,6 @@ mod tests {
|
|||||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||||
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
||||||
#[allow(unsafe_code)]
|
#[allow(unsafe_code)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
||||||
where
|
where
|
||||||
F: FnOnce() -> Fut,
|
F: FnOnce() -> Fut,
|
||||||
|
|||||||
@@ -759,10 +759,6 @@ pub struct ManualTransitionWorkerResultRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ManualTransitionWorkerResultRecord {
|
impl ManualTransitionWorkerResultRecord {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||||
Self::new_with_reason(job_id, task_key, result, None)
|
Self::new_with_reason(job_id, task_key, result, None)
|
||||||
}
|
}
|
||||||
@@ -1261,10 +1257,6 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn load_manual_transition_task_record(
|
pub async fn load_manual_transition_task_record(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1328,10 +1320,6 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn load_manual_transition_worker_result_stats(
|
pub async fn load_manual_transition_worker_result_stats(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1467,10 +1455,6 @@ async fn scan_manual_transition_worker_result_journal(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn reconcile_manual_transition_worker_results(
|
pub async fn reconcile_manual_transition_worker_results(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ pub mod core;
|
|||||||
pub mod evaluator;
|
pub mod evaluator;
|
||||||
pub mod manual_transition_job;
|
pub mod manual_transition_job;
|
||||||
mod metadata_boundary;
|
mod metadata_boundary;
|
||||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||||
mod object_lock_boundary;
|
mod object_lock_boundary;
|
||||||
pub use self::core as lifecycle;
|
pub use self::core as lifecycle;
|
||||||
mod replication_sink;
|
mod replication_sink;
|
||||||
|
|||||||
@@ -15,35 +15,25 @@
|
|||||||
use rustfs_common::metrics::IlmAction;
|
use rustfs_common::metrics::IlmAction;
|
||||||
|
|
||||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
|
||||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||||
pub(crate) use crate::bucket::replication::{
|
pub(crate) use crate::bucket::replication::{
|
||||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||||
};
|
};
|
||||||
|
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||||
use crate::storage_api_contracts::object::DeletedObject;
|
use crate::storage_api_contracts::object::DeletedObject;
|
||||||
|
|
||||||
#[allow(
|
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||||
obj.version_purge_status.is_pending()
|
obj.version_purge_status.is_pending()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,10 +14,6 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -331,10 +331,6 @@ where
|
|||||||
persist_tier_delete_journal_entry(api, &committed).await
|
persist_tier_delete_journal_entry(api, &committed).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||||
where
|
where
|
||||||
S: ObjectOperations<
|
S: ObjectOperations<
|
||||||
|
|||||||
@@ -148,10 +148,6 @@ struct RecoveryCursor {
|
|||||||
object: String,
|
object: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn recover_tier_free_versions(
|
pub async fn recover_tier_free_versions(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
|
|||||||
@@ -80,10 +80,7 @@ impl LastDayTierStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||||
let mut cl = self.clone();
|
let mut cl = self.clone();
|
||||||
let mut cm = m;
|
let mut cm = m;
|
||||||
|
|||||||
@@ -177,10 +177,9 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
#[allow(dead_code)]
|
||||||
struct ObjSweeper {
|
struct ObjSweeper {
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
object: String,
|
object: String,
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
bucket: String,
|
bucket: String,
|
||||||
version_id: Option<Uuid>,
|
version_id: Option<Uuid>,
|
||||||
versioned: bool,
|
versioned: bool,
|
||||||
@@ -192,9 +191,9 @@ struct ObjSweeper {
|
|||||||
remote_object: String,
|
remote_object: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl ObjSweeper {
|
impl ObjSweeper {
|
||||||
#[allow(clippy::new_ret_no_self)]
|
#[allow(clippy::new_ret_no_self)]
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
object: object.into(),
|
object: object.into(),
|
||||||
@@ -203,20 +202,17 @@ impl ObjSweeper {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||||
self.version_id = vid.clone();
|
self.version_id = vid.clone();
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||||
self.versioned = versioned;
|
self.versioned = versioned;
|
||||||
self.suspended = suspended;
|
self.suspended = suspended;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||||
let mut opts = ObjectOpts {
|
let mut opts = ObjectOpts {
|
||||||
version_id: self.version_id.clone(),
|
version_id: self.version_id.clone(),
|
||||||
@@ -230,7 +226,6 @@ impl ObjSweeper {
|
|||||||
opts
|
opts
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||||
self.transition_tier = info.tier;
|
self.transition_tier = info.tier;
|
||||||
self.transition_status = info.status;
|
self.transition_status = info.status;
|
||||||
@@ -271,7 +266,6 @@ impl ObjSweeper {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||||
let Some(je) = self.should_remove_remote_object() else {
|
let Some(je) = self.should_remove_remote_object() else {
|
||||||
return;
|
return;
|
||||||
@@ -391,10 +385,6 @@ impl ExpiryOp for Jentry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||||
if let Err(err) = &result
|
if let Err(err) = &result
|
||||||
@@ -405,10 +395,6 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||||
@@ -419,10 +405,6 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
|||||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
@@ -503,10 +485,6 @@ pub enum RemoteTierDeleteOutcome {
|
|||||||
AlreadyRemoved,
|
AlreadyRemoved,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn delete_object_from_remote_tier_idempotent(
|
pub async fn delete_object_from_remote_tier_idempotent(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
|
|||||||
@@ -50,16 +50,8 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
|||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum TransitionTransactionError {
|
pub enum TransitionTransactionError {
|
||||||
#[error("transition transaction already exists")]
|
#[error("transition transaction already exists")]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
AlreadyExists,
|
AlreadyExists,
|
||||||
#[error("transition transaction is not found")]
|
#[error("transition transaction is not found")]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
|
||||||
)]
|
|
||||||
NotFound,
|
NotFound,
|
||||||
#[error("transition transaction is corrupt: {0}")]
|
#[error("transition transaction is corrupt: {0}")]
|
||||||
Corrupt(&'static str),
|
Corrupt(&'static str),
|
||||||
|
|||||||
@@ -60,14 +60,12 @@ struct ConfigWriteLockProbeState {
|
|||||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
|
||||||
pub struct ConfigWriteLockProbe {
|
pub struct ConfigWriteLockProbe {
|
||||||
state: Arc<ConfigWriteLockProbeState>,
|
state: Arc<ConfigWriteLockProbeState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
impl ConfigWriteLockProbe {
|
impl ConfigWriteLockProbe {
|
||||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
|
||||||
pub fn install(bucket: &str) -> Self {
|
pub fn install(bucket: &str) -> Self {
|
||||||
let state = Arc::new(ConfigWriteLockProbeState {
|
let state = Arc::new(ConfigWriteLockProbeState {
|
||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
@@ -86,7 +84,6 @@ impl ConfigWriteLockProbe {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
|
||||||
pub async fn wait_until_attempted(&self) {
|
pub async fn wait_until_attempted(&self) {
|
||||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
.await
|
.await
|
||||||
@@ -659,16 +656,6 @@ pub async fn update_under_transaction_lock(
|
|||||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
|
||||||
pub async fn delete_under_transaction_lock(
|
|
||||||
guard: &BucketMetadataMutationGuard,
|
|
||||||
bucket: &str,
|
|
||||||
config_file: &str,
|
|
||||||
) -> Result<OffsetDateTime> {
|
|
||||||
guard.ensure_valid(bucket)?;
|
|
||||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn update_quota_if_incarnation(
|
pub async fn update_quota_if_incarnation(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
data: Vec<u8>,
|
data: Vec<u8>,
|
||||||
@@ -808,14 +795,6 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
|||||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
|
||||||
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
|
||||||
bucket: &str,
|
|
||||||
expected_incarnation_id: Uuid,
|
|
||||||
) -> Result<BucketMetadataMutationGuard> {
|
|
||||||
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||||
ctx: &crate::runtime::instance::InstanceContext,
|
ctx: &crate::runtime::instance::InstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -893,10 +872,6 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
|||||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1111,10 +1086,6 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
|||||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1628,7 +1599,6 @@ impl BucketMetadataSys {
|
|||||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||||
/// instead of supplied up front. Loads through this system's own store so
|
/// instead of supplied up front. Loads through this system's own store so
|
||||||
/// the read and the persisted write target the same instance.
|
/// the read and the persisted write target the same instance.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||||
where
|
where
|
||||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||||
@@ -1733,7 +1703,6 @@ impl BucketMetadataSys {
|
|||||||
/// A miss is never published as an authoritative default, and a snapshot
|
/// A miss is never published as an authoritative default, and a snapshot
|
||||||
/// read before delete plus same-name recreation cannot replace the new
|
/// read before delete plus same-name recreation cannot replace the new
|
||||||
/// generation.
|
/// generation.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||||
if is_meta_bucketname(bucket) {
|
if is_meta_bucketname(bucket) {
|
||||||
return Err(Error::other("errInvalidArgument"));
|
return Err(Error::other("errInvalidArgument"));
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
pub mod bandwidth;
|
pub mod bandwidth;
|
||||||
pub mod bucket_target_sys;
|
pub mod bucket_target_sys;
|
||||||
|
|||||||
@@ -136,7 +136,6 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
|||||||
|
|
||||||
/// Check if an object has legal hold enabled.
|
/// Check if an object has legal hold enabled.
|
||||||
/// Returns true if legal hold is ON.
|
/// Returns true if legal hold is ON.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||||
@@ -152,7 +151,6 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||||
/// * `false` if the object is not locked
|
/// * `false` if the object is not locked
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||||
// Delete markers are never locked
|
// Delete markers are never locked
|
||||||
if is_delete_marker {
|
if is_delete_marker {
|
||||||
|
|||||||
@@ -193,7 +193,6 @@ pub enum QuotaError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub struct QuotaErrorResponse {
|
pub struct QuotaErrorResponse {
|
||||||
#[serde(rename = "Code")]
|
#[serde(rename = "Code")]
|
||||||
pub code: String,
|
pub code: String,
|
||||||
@@ -209,7 +208,6 @@ pub struct QuotaErrorResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl QuotaErrorResponse {
|
impl QuotaErrorResponse {
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||||
match quota_error {
|
match quota_error {
|
||||||
QuotaError::QuotaExceeded { .. } => Self {
|
QuotaError::QuotaExceeded { .. } => Self {
|
||||||
@@ -312,7 +310,9 @@ mod tests {
|
|||||||
}
|
}
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct LegacyBucketQuota {
|
struct LegacyBucketQuota {
|
||||||
|
#[allow(dead_code)]
|
||||||
quota: Option<u64>,
|
quota: Option<u64>,
|
||||||
|
#[allow(dead_code)]
|
||||||
quota_type: LegacyQuotaType,
|
quota_type: LegacyQuotaType,
|
||||||
}
|
}
|
||||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||||
|
|||||||
@@ -899,7 +899,6 @@ async fn save_ledger_locked(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn fail_next_quota_ledger_save_for_test() {
|
pub fn fail_next_quota_ledger_save_for_test() {
|
||||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{
|
|||||||
pub(crate) use replication_filemeta_boundary::{
|
pub(crate) use replication_filemeta_boundary::{
|
||||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||||
};
|
};
|
||||||
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||||
@@ -81,6 +81,6 @@ pub use replication_queue_boundary::{
|
|||||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||||
|
|||||||
@@ -37,10 +37,6 @@ impl ReplicationConfigStore {
|
|||||||
com::read_config_limited(api, file, max_bytes).await
|
com::read_config_limited(api, file, max_bytes).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||||
where
|
where
|
||||||
S: ReplicationObjectIO,
|
S: ReplicationObjectIO,
|
||||||
|
|||||||
@@ -24,27 +24,15 @@ use super::replication_storage_boundary::{
|
|||||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||||
|
|
||||||
pub(crate) struct ReplicationLifecycleBridge;
|
pub(crate) struct ReplicationLifecycleBridge;
|
||||||
|
|
||||||
impl ReplicationLifecycleBridge {
|
impl ReplicationLifecycleBridge {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||||
ReplicationConfig::new(Some(config), None)
|
ReplicationConfig::new(Some(config), None)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn has_pending_version_purge(
|
pub(crate) fn has_pending_version_purge(
|
||||||
config: &ReplicationLifecycleConfig,
|
config: &ReplicationLifecycleConfig,
|
||||||
object_name: &str,
|
object_name: &str,
|
||||||
@@ -57,10 +45,6 @@ impl ReplicationLifecycleBridge {
|
|||||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) async fn check_delete_replication(
|
pub(crate) async fn check_delete_replication(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &ObjectToDelete,
|
object: &ObjectToDelete,
|
||||||
@@ -70,10 +54,6 @@ impl ReplicationLifecycleBridge {
|
|||||||
check_replicate_delete(bucket, object, source, opts, None).await
|
check_replicate_delete(bucket, object, source, opts, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||||
let pending_status = decision.pending_status();
|
let pending_status = decision.pending_status();
|
||||||
ReplicationState {
|
ReplicationState {
|
||||||
|
|||||||
@@ -19,33 +19,17 @@ use time::OffsetDateTime;
|
|||||||
use super::replication_error_boundary::Result;
|
use super::replication_error_boundary::Result;
|
||||||
use crate::bucket::msgp_decode;
|
use crate::bucket::msgp_decode;
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) struct ReplicationMsgpCodec;
|
pub(crate) struct ReplicationMsgpCodec;
|
||||||
|
|
||||||
impl ReplicationMsgpCodec {
|
impl ReplicationMsgpCodec {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||||
msgp_decode::read_msgp_ext8_time(rd)
|
msgp_decode::read_msgp_ext8_time(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||||
msgp_decode::skip_msgp_value(rd)
|
msgp_decode::skip_msgp_value(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||||
msgp_decode::write_msgp_time(wr, time)
|
msgp_decode::write_msgp_time(wr, time)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,10 +77,6 @@ impl ReplicationObjectBridge {
|
|||||||
load_delete_request_config_in(ctx, bucket).await
|
load_delete_request_config_in(ctx, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) async fn delete_config_snapshot_in(
|
pub(crate) async fn delete_config_snapshot_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -231,10 +231,6 @@ pub(crate) async fn load_delete_replication_config(
|
|||||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) async fn load_delete_replication_config_in(
|
pub(crate) async fn load_delete_replication_config_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -217,10 +217,6 @@ impl DurableMrfBacklogTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||||
let mut tracker = DurableMrfBacklogTracker {
|
let mut tracker = DurableMrfBacklogTracker {
|
||||||
available: true,
|
available: true,
|
||||||
@@ -716,10 +712,6 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
|||||||
|
|
||||||
// MRF worker lifecycle
|
// MRF worker lifecycle
|
||||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
mrf_stop_tx: Sender<()>,
|
mrf_stop_tx: Sender<()>,
|
||||||
|
|
||||||
// Worker size tracking
|
// Worker size tracking
|
||||||
@@ -948,10 +940,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Resizes worker priority and counts
|
/// Resizes worker priority and counts
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn resize_worker_priority(
|
pub async fn resize_worker_priority(
|
||||||
&self,
|
&self,
|
||||||
pri: ReplicationPriority,
|
pri: ReplicationPriority,
|
||||||
@@ -1192,10 +1180,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Queues an MRF save operation
|
/// Queues an MRF save operation
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||||
}
|
}
|
||||||
@@ -1667,10 +1651,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling regular replication operations
|
/// Worker function for handling regular replication operations
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn add_worker(
|
async fn add_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1684,10 +1664,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling large object replication operations
|
/// Worker function for handling large object replication operations
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn add_large_worker(
|
async fn add_large_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1702,10 +1678,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn add_mrf_worker(
|
async fn add_mrf_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1719,10 +1691,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Delete resync metadata from replication resync state in memory
|
/// Delete resync metadata from replication resync state in memory
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||||
let mut status_map = self.resyncer.status_map.write().await;
|
let mut status_map = self.resyncer.status_map.write().await;
|
||||||
status_map.remove(bucket);
|
status_map.remove(bucket);
|
||||||
|
|||||||
@@ -21,31 +21,11 @@ pub(crate) use rustfs_replication::{
|
|||||||
should_count_head_proxy_failure,
|
should_count_head_proxy_failure,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||||
|
|
||||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||||
|
|||||||
@@ -122,10 +122,6 @@ const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
|||||||
"tcp connect error",
|
"tcp connect error",
|
||||||
];
|
];
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
|
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
|
||||||
|
|
||||||
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
|
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
|
||||||
@@ -332,10 +328,6 @@ fn bounded_resync_max_jobs(value: usize) -> usize {
|
|||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ReplicationResyncer {
|
pub struct ReplicationResyncer {
|
||||||
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
|
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub worker_size: usize,
|
pub worker_size: usize,
|
||||||
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
|
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
|
||||||
resync_admission: Arc<Semaphore>,
|
resync_admission: Arc<Semaphore>,
|
||||||
@@ -552,10 +544,6 @@ impl ReplicationResyncer {
|
|||||||
.is_some_and(|status| status.failed_count > 0)
|
.is_some_and(|status| status.failed_count > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
|
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
|
||||||
where
|
where
|
||||||
S: ReplicationObjectIO,
|
S: ReplicationObjectIO,
|
||||||
|
|||||||
@@ -340,10 +340,6 @@ impl ReplicationStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Site replication update replica statistics
|
/// Site replication update replica statistics
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn sr_update_replica_stat(&self, size: i64) {
|
fn sr_update_replica_stat(&self, size: i64) {
|
||||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -708,12 +704,6 @@ impl ReplicationStats {
|
|||||||
} else {
|
} else {
|
||||||
BucketReplicationStats::new()
|
BucketReplicationStats::new()
|
||||||
};
|
};
|
||||||
// Stamp the serializable failure windows from the live samples: the
|
|
||||||
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
|
||||||
// is what cluster aggregation and the metrics endpoints see.
|
|
||||||
for stat in replication_stats.stats.values_mut() {
|
|
||||||
stat.fail_stats.refresh_windows();
|
|
||||||
}
|
|
||||||
let uptime = if cache.contains_key(bucket) {
|
let uptime = if cache.contains_key(bucket) {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(SystemTime::UNIX_EPOCH)
|
.duration_since(SystemTime::UNIX_EPOCH)
|
||||||
|
|||||||
@@ -15,9 +15,7 @@
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_replication::FailStats;
|
pub(crate) use rustfs_replication::FailStats;
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||||
|
SRMetricsSummary, XferStats,
|
||||||
};
|
};
|
||||||
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||||
// can project the internal stats onto the minio-go response shapes through
|
|
||||||
// the storage_api facade chain.
|
|
||||||
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
|
||||||
|
|||||||
@@ -27,10 +27,8 @@ use rustfs_utils::http::{
|
|||||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
||||||
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
|
||||||
ssec_replication_transport_header,
|
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use time::format_description::well_known::Rfc3339;
|
use time::format_description::well_known::Rfc3339;
|
||||||
@@ -121,27 +119,6 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
|
||||||
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
|
||||||
let key = key.as_bytes();
|
|
||||||
key.len() == prefix.len() + suffix.len()
|
|
||||||
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
|
||||||
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
|
||||||
}
|
|
||||||
|
|
||||||
[
|
|
||||||
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
|
||||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
|
||||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
|
||||||
]
|
|
||||||
.iter()
|
|
||||||
.any(|suffix| {
|
|
||||||
["x-rustfs-", "x-minio-"]
|
|
||||||
.iter()
|
|
||||||
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||||
}
|
}
|
||||||
@@ -199,11 +176,6 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_legacy_source_replication_timestamp_key(key) {
|
|
||||||
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if is_internal_key(key) || is_standard_header(key) {
|
if is_internal_key(key) || is_standard_header(key) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -287,23 +259,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
|
|
||||||
if !tags.is_empty() {
|
if !tags.is_empty() {
|
||||||
put_options.user_tags = tags;
|
put_options.user_tags = tags;
|
||||||
|
put_options.internal.tagging_timestamp =
|
||||||
|
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
||||||
|
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||||
|
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||||
|
} else {
|
||||||
|
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Load the stored tagging timestamp independently of whether any tags
|
|
||||||
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
|
||||||
// key, and the deletion's LWW timestamp must still reach the replica.
|
|
||||||
// With no stored key, fall back to mod_time only while tags exist
|
|
||||||
// (MinIO parity); a tagless object without the key was never tagged and
|
|
||||||
// keeps the epoch default (no header).
|
|
||||||
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
|
||||||
{
|
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
|
||||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
|
||||||
} else if !put_options.user_tags.is_empty() {
|
|
||||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
|
||||||
} else {
|
|
||||||
OffsetDateTime::UNIX_EPOCH
|
|
||||||
};
|
|
||||||
|
|
||||||
let metadata = &*object_info.user_defined;
|
let metadata = &*object_info.user_defined;
|
||||||
|
|
||||||
@@ -319,15 +283,13 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
put_options.cache_control = cache_control.to_string();
|
put_options.cache_control = cache_control.to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
||||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||||
if !retain_until_date.is_empty() {
|
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
|
||||||
}
|
|
||||||
put_options.internal.retention_timestamp =
|
put_options.internal.retention_timestamp =
|
||||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
@@ -732,110 +694,6 @@ mod tests {
|
|||||||
assert!(options.internal.replication_request);
|
assert!(options.internal.replication_request);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// DeleteObjectTagging leaves the object tagless but stamps the
|
|
||||||
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
|
||||||
/// still be loaded (and therefore sent) so the replica can order the
|
|
||||||
/// deletion against concurrent tag edits.
|
|
||||||
#[test]
|
|
||||||
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
|
||||||
let mut metadata = std::collections::HashMap::new();
|
|
||||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
|
||||||
|
|
||||||
let object_info = ObjectInfo {
|
|
||||||
user_defined: Arc::new(metadata),
|
|
||||||
user_tags: Arc::new(String::new()),
|
|
||||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
|
||||||
version_id: Some(Uuid::nil()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
|
||||||
|
|
||||||
assert!(options.user_tags.is_empty());
|
|
||||||
assert_eq!(
|
|
||||||
options.internal.tagging_timestamp,
|
|
||||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
|
||||||
"the stored tagging timestamp must load independently of remaining tags"
|
|
||||||
);
|
|
||||||
|
|
||||||
// A tagless object without the stored key was never tagged: the epoch
|
|
||||||
// default keeps the header unsent.
|
|
||||||
let untagged = ObjectInfo {
|
|
||||||
user_tags: Arc::new(String::new()),
|
|
||||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
|
||||||
version_id: Some(Uuid::nil()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
|
||||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
|
||||||
let legacy_keys = [
|
|
||||||
"x-rustfs-source-replication-tagging-timestamp",
|
|
||||||
"x-rustfs-source-replication-retention-timestamp",
|
|
||||||
"x-rustfs-source-replication-legalhold-timestamp",
|
|
||||||
"x-minio-source-replication-tagging-timestamp",
|
|
||||||
"x-minio-source-replication-retention-timestamp",
|
|
||||||
"x-minio-source-replication-legalhold-timestamp",
|
|
||||||
];
|
|
||||||
let object_info = ObjectInfo {
|
|
||||||
user_defined: Arc::new(
|
|
||||||
legacy_keys
|
|
||||||
.iter()
|
|
||||||
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
|
||||||
|
|
||||||
for legacy_key in legacy_keys {
|
|
||||||
assert!(!options.user_metadata.contains_key(legacy_key));
|
|
||||||
assert_eq!(
|
|
||||||
options
|
|
||||||
.user_metadata
|
|
||||||
.get(&format!("x-amz-meta-{legacy_key}"))
|
|
||||||
.map(String::as_str),
|
|
||||||
Some("2099-01-02T03:04:05Z")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
|
||||||
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
|
||||||
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
|
||||||
let mut metadata = HashMap::from([
|
|
||||||
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
|
||||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
|
||||||
]);
|
|
||||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
|
||||||
let object_info = ObjectInfo {
|
|
||||||
user_defined: Arc::new(metadata),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
|
||||||
|
|
||||||
assert!(options.mode.is_none());
|
|
||||||
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
|
||||||
assert_eq!(
|
|
||||||
options.internal.retention_timestamp,
|
|
||||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
|
||||||
);
|
|
||||||
let headers = options.header();
|
|
||||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
|
||||||
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
|
||||||
assert_eq!(
|
|
||||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
|
||||||
Some("2026-01-02T03:04:05Z")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||||
|
|||||||
@@ -40,14 +40,7 @@ impl ARN {
|
|||||||
|
|
||||||
impl Display for ARN {
|
impl Display for ARN {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
// The `minio` partition is deliberate: madmin-go's ParseARN
|
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||||
// hard-rejects any other partition, so native mc/madmin tooling can
|
|
||||||
// only decode remote-target ARNs minted in this form (backlog#1675
|
|
||||||
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
|
||||||
// readable via the FromStr whitelist below; runtime matching between
|
|
||||||
// targets and replication rules is by full-string equality, so mixed
|
|
||||||
// partitions coexist safely.
|
|
||||||
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,12 +48,7 @@ impl FromStr for ARN {
|
|||||||
type Err = std::io::Error;
|
type Err = std::io::Error;
|
||||||
|
|
||||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
if !s.starts_with("arn:rustfs:") {
|
||||||
// from_str(...).unwrap_or_default()` below never fails, so this is
|
|
||||||
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
|
||||||
// the legacy partition and must stay accepted forever (persisted
|
|
||||||
// bucket-targets.json / replication configs from older releases).
|
|
||||||
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,50 +101,14 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// RustFS commonly generates ARNs with an empty region:
|
/// RustFS commonly generates ARNs with an empty region:
|
||||||
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||||
#[test]
|
#[test]
|
||||||
fn from_str_handles_empty_region_segment() {
|
fn from_str_handles_empty_region_segment() {
|
||||||
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||||
|
|
||||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||||
assert_eq!(parsed.id, "depl-123");
|
assert_eq!(parsed.id, "depl-123");
|
||||||
assert_eq!(parsed.bucket, "bucket-a");
|
assert_eq!(parsed.bucket, "bucket-a");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
|
||||||
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
|
||||||
/// native mc/madmin tooling cannot decode remote-target listings.
|
|
||||||
#[test]
|
|
||||||
fn display_emits_minio_partition() {
|
|
||||||
let arn = ARN::new(
|
|
||||||
BucketTargetType::ReplicationService,
|
|
||||||
"depl-123".to_string(),
|
|
||||||
String::new(),
|
|
||||||
"bucket-a".to_string(),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Persisted bucket-targets.json files from older RustFS releases carry
|
|
||||||
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
|
||||||
#[test]
|
|
||||||
fn from_str_accepts_legacy_rustfs_partition() {
|
|
||||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
|
||||||
|
|
||||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
|
||||||
assert_eq!(parsed.region, "us-east-1");
|
|
||||||
assert_eq!(parsed.id, "depl-123");
|
|
||||||
assert_eq!(parsed.bucket, "bucket-a");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
|
||||||
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
|
||||||
/// would otherwise parse as `type=None`.
|
|
||||||
#[test]
|
|
||||||
fn from_str_rejects_unknown_partition() {
|
|
||||||
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
|
||||||
assert!(ARN::from_str("not-an-arn").is_err());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,10 +59,6 @@ impl fmt::Debug for Credentials {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub enum ServiceType {
|
pub enum ServiceType {
|
||||||
#[default]
|
#[default]
|
||||||
Replication,
|
Replication,
|
||||||
|
|||||||
@@ -73,6 +73,23 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
|||||||
check_bucket_name_common(bucket_name, true)
|
check_bucket_name_common(bucket_name, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
||||||
|
if object_name.len() > 1024 {
|
||||||
|
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
||||||
|
}
|
||||||
|
if !object_name.is_ascii() {
|
||||||
|
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
||||||
|
if object_name.trim().is_empty() {
|
||||||
|
return Err(Error::other("Object name cannot be empty"));
|
||||||
|
}
|
||||||
|
check_valid_object_name_prefix(object_name)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||||
where
|
where
|
||||||
T: for<'xml> xml::Deserialize<'xml>,
|
T: for<'xml> xml::Deserialize<'xml>,
|
||||||
@@ -83,10 +100,6 @@ where
|
|||||||
Ok(ans)
|
Ok(ans)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||||
let mut buf = Vec::with_capacity(256);
|
let mut buf = Vec::with_capacity(256);
|
||||||
{
|
{
|
||||||
@@ -173,27 +186,15 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
|||||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||||
/// accept the full S3 key character set.
|
/// accept the full S3 key character set.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||||
|
|
||||||
/// Client-facing reason for path segments Windows can store but not address
|
/// Client-facing reason for path segments Windows can store but not address
|
||||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||||
|
|
||||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||||
@@ -203,10 +204,6 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
|||||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||||
object.split(['/', '\\']).any(|segment| {
|
object.split(['/', '\\']).any(|segment| {
|
||||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||||
|
|||||||
@@ -90,10 +90,6 @@ impl BucketVersioningSys {
|
|||||||
/// caller's own instance context so a second in-process store never
|
/// caller's own instance context so a second in-process store never
|
||||||
/// answers with the first instance's versioning state; falls back to the
|
/// answers with the first instance's versioning state; falls back to the
|
||||||
/// ambient system when the instance cell is not initialized.
|
/// ambient system when the instance cell is not initialized.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||||
return Ok(VersioningConfiguration::default());
|
return Ok(VersioningConfiguration::default());
|
||||||
|
|||||||
@@ -15,7 +15,6 @@
|
|||||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||||
use crate::disk::error::DiskError;
|
use crate::disk::error::DiskError;
|
||||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||||
use futures::future::join_all;
|
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||||
use std::{
|
use std::{
|
||||||
@@ -656,7 +655,6 @@ async fn list_path_raw_inner(
|
|||||||
errs.push(None);
|
errs.push(None);
|
||||||
}
|
}
|
||||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||||
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut current = MetaCacheEntry::default();
|
let mut current = MetaCacheEntry::default();
|
||||||
@@ -678,21 +676,6 @@ async fn list_path_raw_inner(
|
|||||||
let mut has_err = 0;
|
let mut has_err = 0;
|
||||||
let mut agree = 0;
|
let mut agree = 0;
|
||||||
|
|
||||||
// Start every missing head read in the same round so one stalled
|
|
||||||
// disk cannot multiply the wait budget by the erasure-set width.
|
|
||||||
// Outcomes are still consumed below in stable disk-index order.
|
|
||||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
|
||||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let cancel = &revjob_rx;
|
|
||||||
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
|
||||||
});
|
|
||||||
for (i, outcome) in join_all(concurrent_peeks).await {
|
|
||||||
peek_outcomes[i] = Some(outcome);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (i, r) in readers.iter_mut().enumerate() {
|
for (i, r) in readers.iter_mut().enumerate() {
|
||||||
if errs[i].is_some() {
|
if errs[i].is_some() {
|
||||||
has_err += 1;
|
has_err += 1;
|
||||||
@@ -702,10 +685,7 @@ async fn list_path_raw_inner(
|
|||||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||||
entry
|
entry
|
||||||
} else {
|
} else {
|
||||||
let Some(outcome) = peek_outcomes[i].take() else {
|
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
||||||
return Err(DiskError::Unexpected);
|
|
||||||
};
|
|
||||||
match outcome {
|
|
||||||
PeekOutcome::Ready(res) => {
|
PeekOutcome::Ready(res) => {
|
||||||
if let Some(entry) = res {
|
if let Some(entry) = res {
|
||||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||||
@@ -1315,36 +1295,6 @@ mod tests {
|
|||||||
assert_eq!(err, DiskError::Timeout);
|
assert_eq!(err, DiskError::Timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
|
||||||
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
|
||||||
let peek_timeout = Duration::from_millis(20);
|
|
||||||
let started = tokio::time::Instant::now();
|
|
||||||
let err = list_path_raw(
|
|
||||||
CancellationToken::new(),
|
|
||||||
ListPathRawOptions {
|
|
||||||
disks: vec![None, None, None, None],
|
|
||||||
min_disks: 1,
|
|
||||||
test_reader_behaviors: vec![
|
|
||||||
TestReaderBehavior::Stall,
|
|
||||||
TestReaderBehavior::Stall,
|
|
||||||
TestReaderBehavior::Stall,
|
|
||||||
TestReaderBehavior::Stall,
|
|
||||||
],
|
|
||||||
peek_timeout: Some(peek_timeout),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect_err("all stalled readers should fail the listing");
|
|
||||||
|
|
||||||
assert_eq!(err, DiskError::Timeout);
|
|
||||||
assert_eq!(
|
|
||||||
started.elapsed(),
|
|
||||||
peek_timeout,
|
|
||||||
"reader deadlines must overlap instead of accumulating once per disk"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||||
let entry = MetaCacheEntry {
|
let entry = MetaCacheEntry {
|
||||||
|
|||||||
@@ -229,6 +229,17 @@ pub fn http_resp_to_error_response(
|
|||||||
err_resp
|
err_resp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
||||||
|
ErrorResponse {
|
||||||
|
status_code: StatusCode::BAD_REQUEST,
|
||||||
|
code: S3ErrorCode::InvalidArgument,
|
||||||
|
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
||||||
|
.to_string(),
|
||||||
|
bucket_name: bucket_name.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||||
let msg = format!(
|
let msg = format!(
|
||||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||||
@@ -284,6 +295,16 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
||||||
|
ErrorResponse {
|
||||||
|
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||||
|
code: S3ErrorCode::Custom("APINotSupported".into()),
|
||||||
|
message: message.to_string(),
|
||||||
|
request_id: "rustfs".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct GetRequest {
|
pub struct GetRequest {
|
||||||
pub buffer: Vec<u8>,
|
pub buffer: Vec<u8>,
|
||||||
pub offset: i64,
|
pub offset: i64,
|
||||||
@@ -106,12 +107,11 @@ pub struct GetRequest {
|
|||||||
pub setting_object_info: bool,
|
pub setting_object_info: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct GetResponse {
|
pub struct GetResponse {
|
||||||
pub size: i64,
|
pub size: i64,
|
||||||
//pub error: error,
|
//pub error: error,
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
pub did_read: bool,
|
pub did_read: bool,
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
pub object_info: ObjectInfo,
|
pub object_info: ObjectInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,10 +135,6 @@ impl Object {
|
|||||||
Self { ..Default::default() }
|
Self { ..Default::default() }
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||||
let _ = request.did_offset_change;
|
let _ = request.did_offset_change;
|
||||||
let _ = request.offset;
|
let _ = request.offset;
|
||||||
@@ -154,20 +150,12 @@ impl Object {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||||
self.curr_offset += bytes_read;
|
self.curr_offset += bytes_read;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||||
let mut read_req = GetRequest {
|
let mut read_req = GetRequest {
|
||||||
is_read_op: true,
|
is_read_op: true,
|
||||||
@@ -192,10 +180,6 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let _ = self.do_get_request(&GetRequest {
|
let _ = self.do_get_request(&GetRequest {
|
||||||
@@ -208,10 +192,6 @@ impl Object {
|
|||||||
Ok(self.object_info.clone())
|
Ok(self.object_info.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||||
self.curr_offset = offset;
|
self.curr_offset = offset;
|
||||||
|
|
||||||
@@ -239,10 +219,6 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let seek_req = GetRequest {
|
let seek_req = GetRequest {
|
||||||
@@ -277,10 +253,6 @@ impl Object {
|
|||||||
Ok(self.curr_offset)
|
Ok(self.curr_offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||||
self.is_closed = true;
|
self.is_closed = true;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -20,7 +20,6 @@
|
|||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
@@ -28,6 +27,7 @@ use tracing::warn;
|
|||||||
use crate::client::api_error_response::err_invalid_argument;
|
use crate::client::api_error_response::err_invalid_argument;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct AdvancedGetOptions {
|
pub struct AdvancedGetOptions {
|
||||||
pub replication_delete_marker: bool,
|
pub replication_delete_marker: bool,
|
||||||
pub is_replication_ready_for_delete_marker: bool,
|
pub is_replication_ready_for_delete_marker: bool,
|
||||||
@@ -77,7 +77,7 @@ impl GetObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if self.checksum {
|
if self.checksum {
|
||||||
headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
|
headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
|
||||||
}
|
}
|
||||||
headers
|
headers
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -360,6 +360,7 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct ListObjectsOptions {
|
pub struct ListObjectsOptions {
|
||||||
reverse_versions: bool,
|
reverse_versions: bool,
|
||||||
with_versions: bool,
|
with_versions: bool,
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ use crate::client::{
|
|||||||
api_put_object_common::optimal_part_info,
|
api_put_object_common::optimal_part_info,
|
||||||
api_put_object_multipart::UploadPartParams,
|
api_put_object_multipart::UploadPartParams,
|
||||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
||||||
credentials::SignatureType,
|
credentials::SignatureType,
|
||||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl PutObjectOptions {
|
impl PutObjectOptions {
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn set_match_etag(&mut self, etag: &str) {
|
fn set_match_etag(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||||
@@ -149,7 +149,6 @@ impl PutObjectOptions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn set_match_etag_except(&mut self, etag: &str) {
|
fn set_match_etag_except(&mut self, etag: &str) {
|
||||||
if etag == "*" {
|
if etag == "*" {
|
||||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||||
@@ -260,7 +259,6 @@ impl PutObjectOptions {
|
|||||||
header
|
header
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||||
//if self.checksum.is_set() {
|
//if self.checksum.is_set() {
|
||||||
/*if !self.trailing_header_support {
|
/*if !self.trailing_header_support {
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
|||||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||||
|
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||||
let unknown_size;
|
let unknown_size;
|
||||||
let mut object_size = object_size;
|
let mut object_size = object_size;
|
||||||
|
|||||||
@@ -81,6 +81,18 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct UploadedPartRes {
|
||||||
|
pub error: std::io::Error,
|
||||||
|
pub part_num: i64,
|
||||||
|
pub size: i64,
|
||||||
|
pub part: ObjectPart,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct UploadPartReq {
|
||||||
|
pub part_num: i64,
|
||||||
|
pub part: ObjectPart,
|
||||||
|
}
|
||||||
|
|
||||||
impl TransitionClient {
|
impl TransitionClient {
|
||||||
pub async fn put_object_multipart_stream(
|
pub async fn put_object_multipart_stream(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ pub struct RemoveBucketOptions {
|
|||||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct AdvancedRemoveOptions {
|
pub struct AdvancedRemoveOptions {
|
||||||
pub replication_delete_marker: bool,
|
pub replication_delete_marker: bool,
|
||||||
pub replication_status: ReplicationStatus,
|
pub replication_status: ReplicationStatus,
|
||||||
@@ -464,10 +465,10 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub struct RemoveObjectError {
|
pub struct RemoveObjectError {
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
object_name: String,
|
object_name: String,
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
version_id: String,
|
version_id: String,
|
||||||
err: Option<std::io::Error>,
|
err: Option<std::io::Error>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,10 @@ use crate::client::utils::base64_decode;
|
|||||||
|
|
||||||
use super::transition_api;
|
use super::transition_api;
|
||||||
|
|
||||||
|
pub struct ListAllMyBucketsResult {
|
||||||
|
pub owner: Owner,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
pub struct CommonPrefix {
|
pub struct CommonPrefix {
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
@@ -85,10 +89,6 @@ pub struct ListVersionsResult {
|
|||||||
pub next_version_id_marker: String,
|
pub next_version_id_marker: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub struct ListBucketResult {
|
pub struct ListBucketResult {
|
||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
contents: Vec<transition_api::ObjectInfo>,
|
contents: Vec<transition_api::ObjectInfo>,
|
||||||
@@ -102,10 +102,6 @@ pub struct ListBucketResult {
|
|||||||
prefix: String,
|
prefix: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub struct ListMultipartUploadsResult {
|
pub struct ListMultipartUploadsResult {
|
||||||
bucket: String,
|
bucket: String,
|
||||||
key_marker: String,
|
key_marker: String,
|
||||||
@@ -121,15 +117,16 @@ pub struct ListMultipartUploadsResult {
|
|||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub struct Initiator {
|
pub struct Initiator {
|
||||||
id: String,
|
id: String,
|
||||||
display_name: String,
|
display_name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct CopyObjectResult {
|
||||||
|
pub etag: String,
|
||||||
|
pub last_modified: OffsetDateTime,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ObjectPart {
|
pub struct ObjectPart {
|
||||||
pub etag: String,
|
pub etag: String,
|
||||||
@@ -263,7 +260,6 @@ pub struct CompletePart {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CompletePart {
|
impl CompletePart {
|
||||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
|
||||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||||
match t {
|
match t {
|
||||||
ChecksumMode::ChecksumCRC32C => {
|
ChecksumMode::ChecksumCRC32C => {
|
||||||
@@ -288,6 +284,11 @@ impl CompletePart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct CopyObjectPartResult {
|
||||||
|
pub etag: String,
|
||||||
|
pub last_modified: OffsetDateTime,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, serde::Serialize)]
|
#[derive(Debug, Default, serde::Serialize)]
|
||||||
#[serde(rename = "CompleteMultipartUpload")]
|
#[serde(rename = "CompleteMultipartUpload")]
|
||||||
pub struct CompleteMultipartUpload {
|
pub struct CompleteMultipartUpload {
|
||||||
@@ -356,10 +357,10 @@ impl CompleteMultipartUpload {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
pub struct CreateBucketConfiguration {
|
||||||
dead_code,
|
pub location: String,
|
||||||
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
}
|
||||||
)]
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteObject {
|
pub struct DeleteObject {
|
||||||
//api has
|
//api has
|
||||||
@@ -367,6 +368,21 @@ pub struct DeleteObject {
|
|||||||
pub version_id: String,
|
pub version_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct DeletedObject {
|
||||||
|
//s3s has
|
||||||
|
pub key: String,
|
||||||
|
pub version_id: String,
|
||||||
|
pub deletemarker: bool,
|
||||||
|
pub deletemarker_version_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NonDeletedObject {
|
||||||
|
pub key: String,
|
||||||
|
pub code: String,
|
||||||
|
pub message: String,
|
||||||
|
pub version_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteMultiObjects {
|
pub struct DeleteMultiObjects {
|
||||||
pub quiet: bool,
|
pub quiet: bool,
|
||||||
@@ -386,7 +402,6 @@ impl DeleteMultiObjects {
|
|||||||
Ok(buf)
|
Ok(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
|
||||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct WireDeleteObject {
|
struct WireDeleteObject {
|
||||||
@@ -421,3 +436,8 @@ impl DeleteMultiObjects {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct DeleteMultiObjectsResult {
|
||||||
|
pub deleted_objects: Vec<DeletedObject>,
|
||||||
|
pub undeleted_objects: Vec<NonDeletedObject>,
|
||||||
|
}
|
||||||
|
|||||||
@@ -365,15 +365,11 @@ mod tests {
|
|||||||
pub struct Checksum {
|
pub struct Checksum {
|
||||||
checksum_type: ChecksumMode,
|
checksum_type: ChecksumMode,
|
||||||
r: Vec<u8>,
|
r: Vec<u8>,
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
|
||||||
)]
|
|
||||||
computed: bool,
|
computed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl Checksum {
|
impl Checksum {
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||||
return Checksum {
|
return Checksum {
|
||||||
@@ -385,7 +381,7 @@ impl Checksum {
|
|||||||
Checksum::default()
|
Checksum::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||||
let b = match base64_decode(s.as_bytes()) {
|
let b = match base64_decode(s.as_bytes()) {
|
||||||
Ok(b) => b,
|
Ok(b) => b,
|
||||||
@@ -412,7 +408,7 @@ impl Checksum {
|
|||||||
base64_encode(&self.r)
|
base64_encode(&self.r)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
fn raw(&self) -> Option<Vec<u8>> {
|
fn raw(&self) -> Option<Vec<u8>> {
|
||||||
if !self.is_set() {
|
if !self.is_set() {
|
||||||
return None;
|
return None;
|
||||||
|
|||||||
@@ -32,5 +32,8 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
|||||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||||
|
|
||||||
|
pub const TOTAL_WORKERS: i64 = 4;
|
||||||
|
|
||||||
|
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||||
|
|||||||
@@ -67,10 +67,6 @@ impl<P: Provider + Default> Credentials<P> {
|
|||||||
Ok(self.creds.clone())
|
Ok(self.creds.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn expire(&mut self) {
|
fn expire(&mut self) {
|
||||||
self.force_refresh = true;
|
self.force_refresh = true;
|
||||||
}
|
}
|
||||||
@@ -137,10 +133,6 @@ impl Provider for Static {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct STSError {
|
pub struct STSError {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub r#type: String,
|
pub r#type: String,
|
||||||
pub code: String,
|
pub code: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
@@ -149,10 +141,6 @@ pub struct STSError {
|
|||||||
#[derive(Debug, Clone, thiserror::Error)]
|
#[derive(Debug, Clone, thiserror::Error)]
|
||||||
pub struct ErrorResponse {
|
pub struct ErrorResponse {
|
||||||
pub sts_error: STSError,
|
pub sts_error: STSError,
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub request_id: String,
|
pub request_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,3 +158,22 @@ impl ErrorResponse {
|
|||||||
return self.sts_error.message.clone();
|
return self.sts_error.message.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
||||||
|
where
|
||||||
|
for<'de> T: Deserialize<'de>,
|
||||||
|
{
|
||||||
|
match std::str::from_utf8(body) {
|
||||||
|
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||||
|
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
||||||
|
where
|
||||||
|
for<'de> T: Deserialize<'de>,
|
||||||
|
{
|
||||||
|
let body = body_reader.to_vec();
|
||||||
|
let parsed = xml_decoder(&body)?;
|
||||||
|
Ok((body, parsed))
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
pub mod admin_handler_utils;
|
pub mod admin_handler_utils;
|
||||||
pub mod api_error_response;
|
pub mod api_error_response;
|
||||||
|
|||||||
@@ -37,17 +37,16 @@ pub struct PutObjReader {
|
|||||||
//pub sealMD5Fn: SealMD5CurrFn,
|
//pub sealMD5Fn: SealMD5CurrFn,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl PutObjReader {
|
impl PutObjReader {
|
||||||
pub fn new(reader: HashReader) -> Self {
|
pub fn new(reader: HashReader) -> Self {
|
||||||
Self { reader }
|
Self { reader }
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn md5_current_hex_string(&self) -> String {
|
fn md5_current_hex_string(&self) -> String {
|
||||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||||
self.reader = enc_reader;
|
self.reader = enc_reader;
|
||||||
|
|
||||||
@@ -78,6 +77,39 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
||||||
|
let mut skip_length: i64 = 0;
|
||||||
|
let mut cumulative_actual_size: i64 = 0;
|
||||||
|
let mut first_part_idx: i64 = 0;
|
||||||
|
let mut compressed_offset: i64 = 0;
|
||||||
|
let mut part_skip: i64 = 0;
|
||||||
|
let mut decrypt_skip: i64 = 0;
|
||||||
|
let mut seq_num: u64 = 0;
|
||||||
|
for (i, part) in oi.parts.iter().enumerate() {
|
||||||
|
cumulative_actual_size += part.actual_size as i64;
|
||||||
|
if cumulative_actual_size <= offset {
|
||||||
|
compressed_offset += part.size as i64;
|
||||||
|
} else {
|
||||||
|
first_part_idx = i as i64;
|
||||||
|
skip_length = cumulative_actual_size - part.actual_size as i64;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
skip_length = offset - skip_length;
|
||||||
|
|
||||||
|
let parts: &[ObjectPartInfo] = &oi.parts;
|
||||||
|
if skip_length > 0
|
||||||
|
&& parts.len() > first_part_idx as usize
|
||||||
|
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
||||||
|
{
|
||||||
|
let _ = part_skip;
|
||||||
|
let _ = decrypt_skip;
|
||||||
|
let _ = seq_num;
|
||||||
|
}
|
||||||
|
|
||||||
|
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn new_getobjectreader<'a>(
|
pub fn new_getobjectreader<'a>(
|
||||||
rs: &Option<HTTPRangeSpec>,
|
rs: &Option<HTTPRangeSpec>,
|
||||||
oi: &'a ObjectInfo,
|
oi: &'a ObjectInfo,
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
|
|||||||
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
|
||||||
pub(crate) enum BucketVersioningState {
|
pub(crate) enum BucketVersioningState {
|
||||||
Unknown,
|
Unknown,
|
||||||
Disabled,
|
Disabled,
|
||||||
@@ -48,7 +47,6 @@ impl RemoteVersion {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
|
||||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||||
match self {
|
match self {
|
||||||
Self::Unknown => Err(Error::new(
|
Self::Unknown => Err(Error::new(
|
||||||
|
|||||||
@@ -54,10 +54,6 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
|||||||
use rustfs_rio::HashReader;
|
use rustfs_rio::HashReader;
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::{
|
use rustfs_utils::{
|
||||||
http::headers::{
|
|
||||||
AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1,
|
|
||||||
AMZ_CHECKSUM_SHA256,
|
|
||||||
},
|
|
||||||
net::get_endpoint_url,
|
net::get_endpoint_url,
|
||||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||||
};
|
};
|
||||||
@@ -105,10 +101,6 @@ where
|
|||||||
|
|
||||||
const C_UNKNOWN: i32 = -1;
|
const C_UNKNOWN: i32 = -1;
|
||||||
const C_OFFLINE: i32 = 0;
|
const C_OFFLINE: i32 = 0;
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reachable only from the unused transition client methods below (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const C_ONLINE: i32 = 1;
|
const C_ONLINE: i32 = 1;
|
||||||
|
|
||||||
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
||||||
@@ -328,10 +320,6 @@ impl TransitionClient {
|
|||||||
Ok(client)
|
Ok(client)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn endpoint_url(&self) -> Url {
|
fn endpoint_url(&self) -> Url {
|
||||||
self.endpoint_url.clone()
|
self.endpoint_url.clone()
|
||||||
}
|
}
|
||||||
@@ -360,20 +348,12 @@ impl TransitionClient {
|
|||||||
.to_string())
|
.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn trace_errors_only_off(&self) {
|
fn trace_errors_only_off(&self) {
|
||||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||||
*trace_errors_only = false;
|
*trace_errors_only = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn trace_off(&self) {
|
fn trace_off(&self) {
|
||||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||||
*is_trace_enabled = false;
|
*is_trace_enabled = false;
|
||||||
@@ -383,20 +363,12 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||||
*endpoint = accelerate_endpoint.to_string();
|
*endpoint = accelerate_endpoint.to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||||
*dual_stack = enabled;
|
*dual_stack = enabled;
|
||||||
@@ -426,18 +398,10 @@ impl TransitionClient {
|
|||||||
(hash_algos, hash_sums)
|
(hash_algos, hash_sums)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn is_online(&self) -> bool {
|
fn is_online(&self) -> bool {
|
||||||
!self.is_offline()
|
!self.is_offline()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn mark_offline(&self) {
|
fn mark_offline(&self) {
|
||||||
self.health_status
|
self.health_status
|
||||||
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
||||||
@@ -447,18 +411,10 @@ impl TransitionClient {
|
|||||||
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn health_check(hc_duration: Duration) {
|
fn health_check(hc_duration: Duration) {
|
||||||
let _ = hc_duration;
|
let _ = hc_duration;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
||||||
let mut resp_trace: Vec<u8>;
|
let mut resp_trace: Vec<u8>;
|
||||||
|
|
||||||
@@ -1146,7 +1102,6 @@ impl Default for ObjectInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectInfo {
|
impl ObjectInfo {
|
||||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
|
||||||
pub(crate) fn remote_version(
|
pub(crate) fn remote_version(
|
||||||
&self,
|
&self,
|
||||||
capabilities: ProviderVersionCapabilities,
|
capabilities: ProviderVersionCapabilities,
|
||||||
@@ -1387,12 +1342,12 @@ pub(crate) fn to_object_info_for_provider(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Extract checksums
|
// Extract checksums
|
||||||
let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32);
|
let checksum_crc32 = get_header("x-amz-checksum-crc32");
|
||||||
let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C);
|
let checksum_crc32c = get_header("x-amz-checksum-crc32c");
|
||||||
let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1);
|
let checksum_sha1 = get_header("x-amz-checksum-sha1");
|
||||||
let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256);
|
let checksum_sha256 = get_header("x-amz-checksum-sha256");
|
||||||
let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME);
|
let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
|
||||||
let checksum_mode = get_header(AMZ_CHECKSUM_MODE);
|
let checksum_mode = get_header("x-amz-checksum-mode");
|
||||||
|
|
||||||
// Build and return the ObjectInfo struct
|
// Build and return the ObjectInfo struct
|
||||||
Ok(ObjectInfo {
|
Ok(ObjectInfo {
|
||||||
|
|||||||
@@ -48,6 +48,10 @@ lazy_static! {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
||||||
|
SUPPORTED_QUERY_VALUES[qs_key]
|
||||||
|
}
|
||||||
|
|
||||||
pub fn is_storageclass_header(header_key: &str) -> bool {
|
pub fn is_storageclass_header(header_key: &str) -> bool {
|
||||||
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,25 +86,6 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
|||||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
/// Error for a peer that reported `success = false` without an `error_info` payload.
|
|
||||||
///
|
|
||||||
/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError`
|
|
||||||
/// instead of `DiskError`. The message names the operation (and the bucket, where the
|
|
||||||
/// operation has one) and nothing else, for two reasons:
|
|
||||||
///
|
|
||||||
/// - `finalize_result` classifies failures by message substring, so any text matching
|
|
||||||
/// `message_has_network_needle` would take an answering peer offline and evict its
|
|
||||||
/// connection over a plain application-level rejection.
|
|
||||||
/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered
|
|
||||||
/// message, so a per-peer detail such as the peer address would split one shared
|
|
||||||
/// failure into single-count buckets and downgrade the dominant error.
|
|
||||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
|
||||||
match bucket {
|
|
||||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
|
||||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
||||||
if !response.success {
|
if !response.success {
|
||||||
return Err(Error::other(
|
return Err(Error::other(
|
||||||
@@ -715,7 +696,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("local_storage_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.storage_info;
|
let data = response.storage_info;
|
||||||
|
|
||||||
@@ -738,7 +719,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("server_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.server_properties;
|
let data = response.server_properties;
|
||||||
|
|
||||||
@@ -761,7 +742,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_cpus", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.cpus;
|
let data = response.cpus;
|
||||||
|
|
||||||
@@ -784,7 +765,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_net_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.net_info;
|
let data = response.net_info;
|
||||||
|
|
||||||
@@ -807,7 +788,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_partitions", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.partitions;
|
let data = response.partitions;
|
||||||
|
|
||||||
@@ -830,7 +811,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_os_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.os_info;
|
let data = response.os_info;
|
||||||
|
|
||||||
@@ -851,7 +832,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_se_linux_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.sys_services;
|
let data = response.sys_services;
|
||||||
|
|
||||||
@@ -876,7 +857,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_sys_config", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.sys_config;
|
let data = response.sys_config;
|
||||||
|
|
||||||
@@ -901,7 +882,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_sys_errors", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.sys_errors;
|
let data = response.sys_errors;
|
||||||
|
|
||||||
@@ -926,7 +907,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_mem_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.mem_info;
|
let data = response.mem_info;
|
||||||
|
|
||||||
@@ -958,7 +939,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_metrics", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.realtime_metrics;
|
let data = response.realtime_metrics;
|
||||||
|
|
||||||
@@ -983,7 +964,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_live_events", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(PeerLiveEventsBatch {
|
Ok(PeerLiveEventsBatch {
|
||||||
@@ -1008,7 +989,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("get_proc_info", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
let data = response.proc_info;
|
let data = response.proc_info;
|
||||||
|
|
||||||
@@ -1035,7 +1016,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("start_profiling", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1342,7 +1323,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1365,7 +1346,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket)));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1388,7 +1369,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("delete_policy", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1411,7 +1392,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_policy", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1436,7 +1417,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_policy_mapping", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1459,7 +1440,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("delete_user", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1482,7 +1463,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("delete_service_account", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1506,7 +1487,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_user", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1529,7 +1510,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_service_account", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1552,7 +1533,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_group", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1573,7 +1554,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("reload_site_replication_config", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1616,7 +1597,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("signal_service", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
||||||
Ok(response)
|
Ok(response)
|
||||||
@@ -1686,7 +1667,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("reload_pool_meta", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1710,7 +1691,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("stop_rebalance", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1744,7 +1725,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("load_rebalance_meta", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1772,7 +1753,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("start_decommission", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1796,7 +1777,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("decommission_cancel", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1820,7 +1801,7 @@ impl PeerRestClient {
|
|||||||
if let Some(msg) = response.error_info {
|
if let Some(msg) = response.error_info {
|
||||||
return Err(Error::other(msg));
|
return Err(Error::other(msg));
|
||||||
}
|
}
|
||||||
return Err(peer_failure_without_details("clear_decommission", None));
|
return Err(Error::other(""));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1966,8 +1947,6 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||||
use crate::disk::error::DiskError;
|
|
||||||
use crate::disk::error_reduce::reduce_errs;
|
|
||||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -3119,115 +3098,4 @@ mod tests {
|
|||||||
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every operation name passed to `peer_failure_without_details` in this file.
|
|
||||||
const PEER_FAILURE_OPS: &[&str] = &[
|
|
||||||
"local_storage_info",
|
|
||||||
"server_info",
|
|
||||||
"get_cpus",
|
|
||||||
"get_net_info",
|
|
||||||
"get_partitions",
|
|
||||||
"get_os_info",
|
|
||||||
"get_se_linux_info",
|
|
||||||
"get_sys_config",
|
|
||||||
"get_sys_errors",
|
|
||||||
"get_mem_info",
|
|
||||||
"get_metrics",
|
|
||||||
"get_live_events",
|
|
||||||
"get_proc_info",
|
|
||||||
"start_profiling",
|
|
||||||
"load_bucket_metadata",
|
|
||||||
"delete_bucket_metadata",
|
|
||||||
"delete_policy",
|
|
||||||
"load_policy",
|
|
||||||
"load_policy_mapping",
|
|
||||||
"delete_user",
|
|
||||||
"delete_service_account",
|
|
||||||
"load_user",
|
|
||||||
"load_service_account",
|
|
||||||
"load_group",
|
|
||||||
"reload_site_replication_config",
|
|
||||||
"signal_service",
|
|
||||||
"reload_pool_meta",
|
|
||||||
"stop_rebalance",
|
|
||||||
"load_rebalance_meta",
|
|
||||||
"start_decommission",
|
|
||||||
"decommission_cancel",
|
|
||||||
"clear_decommission",
|
|
||||||
];
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
|
||||||
for op in PEER_FAILURE_OPS {
|
|
||||||
let message = peer_failure_without_details(op, None).to_string();
|
|
||||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
|
||||||
}
|
|
||||||
|
|
||||||
for op in ["load_bucket_metadata", "delete_bucket_metadata"] {
|
|
||||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
|
||||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
|
||||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
|
||||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
|
||||||
// same operation must stay a single dominant error instead of one bucket per peer.
|
|
||||||
let per_peer_errs = (0..4)
|
|
||||||
.map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let (count, dominant) = reduce_errs(&per_peer_errs, &[]);
|
|
||||||
assert_eq!(count, 4, "one shared failure must not split into per-peer buckets");
|
|
||||||
assert_eq!(
|
|
||||||
dominant,
|
|
||||||
Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_ne!(
|
|
||||||
peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(),
|
|
||||||
peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string()
|
|
||||||
);
|
|
||||||
assert_ne!(
|
|
||||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(),
|
|
||||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn peer_failure_without_details_never_reads_as_a_network_failure() {
|
|
||||||
// `finalize_result` marks the peer offline and evicts its connection whenever the
|
|
||||||
// message matches a network needle. A peer that answered `success = false` is alive,
|
|
||||||
// so no operation or bucket name may push this text over that classifier.
|
|
||||||
for op in PEER_FAILURE_OPS {
|
|
||||||
let err = peer_failure_without_details(op, None);
|
|
||||||
assert!(
|
|
||||||
!PeerRestClient::is_network_like_error(&err),
|
|
||||||
"{op} must not read as a transport failure: {err}"
|
|
||||||
);
|
|
||||||
|
|
||||||
let scoped = peer_failure_without_details(op, Some("bucket-name"));
|
|
||||||
assert!(
|
|
||||||
!PeerRestClient::is_network_like_error(&scoped),
|
|
||||||
"{op} must not read as a transport failure: {scoped}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// The bucket name is caller-supplied. Every needle carries a space, which S3 bucket
|
|
||||||
// names cannot, and the name is closed by `)` before the literal text resumes, so no
|
|
||||||
// needle can straddle the boundary either.
|
|
||||||
for bucket in [
|
|
||||||
"timed-out",
|
|
||||||
"connection-reset",
|
|
||||||
"transport-error",
|
|
||||||
"broken-pipe",
|
|
||||||
"unavailable-logs",
|
|
||||||
] {
|
|
||||||
let err = peer_failure_without_details("load_bucket_metadata", Some(bucket));
|
|
||||||
assert!(
|
|
||||||
!PeerRestClient::is_network_like_error(&err),
|
|
||||||
"bucket {bucket} must not push the message over the network classifier: {err}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -214,21 +214,6 @@ fn pool_write_quorum(participant_count: usize) -> usize {
|
|||||||
(participant_count / 2) + 1
|
(participant_count / 2) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Error for a peer that reported `success = false` without an error payload.
|
|
||||||
///
|
|
||||||
/// The message must stay identical across the peers of one operation: `reduce_errs`
|
|
||||||
/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address,
|
|
||||||
/// timing) would split one shared failure into single-count buckets and downgrade a real
|
|
||||||
/// dominant error into `ErasureWriteQuorum`.
|
|
||||||
///
|
|
||||||
/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape.
|
|
||||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
|
||||||
match bucket {
|
|
||||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
|
||||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
||||||
if per_pool_errs.is_empty() {
|
if per_pool_errs.is_empty() {
|
||||||
return Some(Error::ErasureWriteQuorum);
|
return Some(Error::ErasureWriteQuorum);
|
||||||
@@ -1093,7 +1078,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(peer_failure_without_details("heal_bucket", Some(bucket)))
|
Err(Error::other(""))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1120,7 +1105,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(peer_failure_without_details("list_bucket", None))
|
Err(Error::other(""))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let bucket_infos = response
|
let bucket_infos = response
|
||||||
@@ -1151,7 +1136,9 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(peer_failure_without_details("make_bucket", Some(bucket)))
|
Err(Error::other(format!(
|
||||||
|
"make_bucket({bucket}): peer returned failure without error details"
|
||||||
|
)))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1175,7 +1162,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(peer_failure_without_details("get_bucket_info", Some(bucket)))
|
Err(Error::other(""))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
||||||
@@ -1203,7 +1190,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
|||||||
return if let Some(err) = response.error {
|
return if let Some(err) = response.error {
|
||||||
Err(err.into())
|
Err(err.into())
|
||||||
} else {
|
} else {
|
||||||
Err(peer_failure_without_details("delete_bucket", Some(bucket)))
|
Err(Error::other(""))
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2327,37 +2314,4 @@ mod tests {
|
|||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
|
||||||
for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] {
|
|
||||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
|
||||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
|
||||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
|
||||||
}
|
|
||||||
|
|
||||||
let message = peer_failure_without_details("list_bucket", None).to_string();
|
|
||||||
assert!(message.contains("list_bucket"), "cluster-wide message must name the operation");
|
|
||||||
assert!(!message.trim().is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
|
||||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
|
||||||
// same operation on the same bucket must still reach quorum as one dominant error.
|
|
||||||
let per_pool_errs = vec![
|
|
||||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
|
||||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
|
||||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
|
||||||
];
|
|
||||||
assert_eq!(
|
|
||||||
reduce_pool_write_quorum_errs(&per_pool_errs),
|
|
||||||
Some(peer_failure_without_details("delete_bucket", Some("shared")))
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_ne!(
|
|
||||||
peer_failure_without_details("delete_bucket", Some("shared")),
|
|
||||||
peer_failure_without_details("get_bucket_info", Some("shared"))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ use rustfs_config::{
|
|||||||
};
|
};
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
#[allow(clippy::declare_interior_mutable_const)]
|
#[allow(clippy::declare_interior_mutable_const)]
|
||||||
/// Default KVS for audit webhook settings.
|
/// Default KVS for audit webhook settings.
|
||||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
@@ -116,6 +117,7 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
])
|
])
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
#[allow(clippy::declare_interior_mutable_const)]
|
#[allow(clippy::declare_interior_mutable_const)]
|
||||||
/// Default KVS for audit MQTT settings.
|
/// Default KVS for audit MQTT settings.
|
||||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
@@ -373,6 +375,7 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
])
|
])
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
KVS(vec![
|
KVS(vec![
|
||||||
KV {
|
KV {
|
||||||
|
|||||||
@@ -12,9 +12,12 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
use crate::error::{Error, Result};
|
||||||
use rustfs_config::server_config::{KV, KVS};
|
use rustfs_config::server_config::{KV, KVS};
|
||||||
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
||||||
|
use rustfs_utils::string::parse_bool;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||||
KVS(vec![KV {
|
KVS(vec![KV {
|
||||||
@@ -23,3 +26,59 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
hidden_if_empty: false,
|
hidden_if_empty: false,
|
||||||
}])
|
}])
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct Config {
|
||||||
|
pub bitrot: String,
|
||||||
|
pub sleep: Duration,
|
||||||
|
pub io_count: usize,
|
||||||
|
pub drive_workers: usize,
|
||||||
|
pub cache: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Config {
|
||||||
|
pub fn bitrot_scan_cycle(&self) -> Duration {
|
||||||
|
self.cache
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_workers(&self) -> usize {
|
||||||
|
self.drive_workers
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update(&mut self, nopts: &Config) {
|
||||||
|
self.bitrot = nopts.bitrot.clone();
|
||||||
|
self.io_count = nopts.io_count;
|
||||||
|
self.sleep = nopts.sleep;
|
||||||
|
self.drive_workers = nopts.drive_workers;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
||||||
|
|
||||||
|
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
||||||
|
match parse_bool(s) {
|
||||||
|
Ok(enabled) => {
|
||||||
|
if enabled {
|
||||||
|
Ok(Duration::from_secs_f64(0.0))
|
||||||
|
} else {
|
||||||
|
Ok(Duration::from_secs_f64(-1.0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
if !s.ends_with("m") {
|
||||||
|
return Err(Error::other("unknown format"));
|
||||||
|
}
|
||||||
|
|
||||||
|
match s.trim_end_matches('m').parse::<u64>() {
|
||||||
|
Ok(months) => {
|
||||||
|
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
||||||
|
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
||||||
|
}
|
||||||
|
Err(err) => Err(Error::other(err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
|
|
||||||
mod audit;
|
mod audit;
|
||||||
pub mod com;
|
pub mod com;
|
||||||
|
#[allow(dead_code)]
|
||||||
pub mod heal;
|
pub mod heal;
|
||||||
mod notify;
|
mod notify;
|
||||||
mod oidc;
|
mod oidc;
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ use crate::bucket::replication::replication_state_from_filemeta;
|
|||||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||||
use crate::bucket::{
|
use crate::bucket::{
|
||||||
lifecycle::{
|
lifecycle::{
|
||||||
LifecycleExpiryConfigs,
|
|
||||||
bucket_lifecycle_audit::LcEventSrc,
|
bucket_lifecycle_audit::LcEventSrc,
|
||||||
bucket_lifecycle_ops::{
|
bucket_lifecycle_ops::{
|
||||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||||
@@ -1997,11 +1996,11 @@ impl PoolMeta {
|
|||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||||
struct PoolInfo {
|
struct PoolInfo {
|
||||||
position: usize,
|
position: usize,
|
||||||
completed: bool,
|
completed: bool,
|
||||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
|
||||||
decom_started: bool,
|
decom_started: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2336,10 +2335,6 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
|
|
||||||
action.delete()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
||||||
if !apply_actions || applied {
|
if !apply_actions || applied {
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
@@ -2390,80 +2385,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct HealLifecycleExpiryContext {
|
|
||||||
configs: LifecycleExpiryConfigs,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ECStore {
|
impl ECStore {
|
||||||
pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
|
||||||
if bucket == RUSTFS_META_BUCKET {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
let configs = get_expiry_configs(self, bucket).await?;
|
|
||||||
if configs.lifecycle.is_none() {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Some(HealLifecycleExpiryContext { configs }))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn enqueue_heal_lifecycle_expiry(
|
|
||||||
self: &Arc<Self>,
|
|
||||||
context: &HealLifecycleExpiryContext,
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
version_id: Option<&str>,
|
|
||||||
object_info: Option<&crate::object_api::ObjectInfo>,
|
|
||||||
) -> Result<bool> {
|
|
||||||
let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
|
|
||||||
return Ok(false);
|
|
||||||
};
|
|
||||||
|
|
||||||
let object_info = if let Some(object_info) = object_info {
|
|
||||||
if object_info.bucket != bucket || object_info.name != object {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
let snapshot_version_id = object_info
|
|
||||||
.version_id
|
|
||||||
.filter(|version_id| !version_id.is_nil())
|
|
||||||
.map(|version_id| version_id.to_string());
|
|
||||||
if snapshot_version_id.as_deref() != version_id {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
object_info.clone()
|
|
||||||
} else {
|
|
||||||
match self
|
|
||||||
.get_object_info(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&ObjectOptions {
|
|
||||||
version_id: version_id.map(str::to_string),
|
|
||||||
versioned: version_id.is_some(),
|
|
||||||
expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(object_info) => object_info,
|
|
||||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
|
|
||||||
Err(err) => return Err(err),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
|
|
||||||
if !lifecycle_action_skips_heal_version(event.action) {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn save_current_pool_meta(&self) -> Result<()> {
|
async fn save_current_pool_meta(&self) -> Result<()> {
|
||||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||||
let snapshot = {
|
let snapshot = {
|
||||||
@@ -4365,19 +4287,6 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn lifecycle_action_skips_heal_version_for_every_delete_action() {
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
|
|
||||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
|
|
||||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
|
|
||||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
|
|
||||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
||||||
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
||||||
@@ -5049,19 +4958,13 @@ fn is_disk_online_state(state: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||||
fallback_total_capacity_dedup(disks)
|
fallback_total_capacity_dedup(disks)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||||
fallback_free_capacity_dedup(disks)
|
fallback_free_capacity_dedup(disks)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
|||||||
|
|
||||||
Err(Error::DiskNotFound)
|
Err(Error::DiskNotFound)
|
||||||
}
|
}
|
||||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||||
self.get_disks_for_heal_object(object, opts)?
|
// Multipart orphan reconciliation is intentionally retained above the pool/set layers
|
||||||
.check_abandoned_parts(bucket, object, opts)
|
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||||
.await
|
Err(StorageError::NotImplemented)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1996,7 +1996,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sets_check_abandoned_parts_rejects_invalid_set_scope() {
|
async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() {
|
||||||
let format = FormatV3::new(1, 1);
|
let format = FormatV3::new(1, 1);
|
||||||
let sets = Sets {
|
let sets = Sets {
|
||||||
id: format.id,
|
id: format.id,
|
||||||
@@ -2021,21 +2021,10 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let err = sets
|
let err = sets
|
||||||
.check_abandoned_parts(
|
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||||
"bucket",
|
|
||||||
"object",
|
|
||||||
&HealOpts {
|
|
||||||
set: Some(1),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.expect_err("out-of-range abandoned-parts set scope must fail closed");
|
.expect_err("abandoned-parts ownership should stay above the pool/set storage layers");
|
||||||
assert!(
|
assert!(matches!(err, StorageError::NotImplemented));
|
||||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
|
||||||
if field == "set" && reason.contains("invalid heal set index 1")),
|
|
||||||
"unexpected invalid set error: {err:?}"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
||||||
|
|||||||
@@ -190,17 +190,6 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
|
|||||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
|
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
||||||
@@ -562,32 +551,6 @@ mod tests {
|
|||||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
||||||
assert_eq!(
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
|
||||||
"data_read_inline_body_verify"
|
|
||||||
);
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
|
|
||||||
assert_eq!(
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
|
||||||
"data_read_inline_identity_mismatch"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
|
||||||
"data_read_inline_missing_payload"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
|
|
||||||
"data_read_inline_missing_shard"
|
|
||||||
);
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
|
|
||||||
assert_eq!(
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
|
||||||
"data_read_inline_transformed"
|
|
||||||
);
|
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
||||||
|
|||||||
@@ -637,23 +637,14 @@ impl Default for DiskOperationMetrics {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl DiskOperationMetrics {
|
impl DiskOperationMetrics {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn record_call(&mut self) {
|
fn record_call(&mut self) {
|
||||||
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_latency_atomic(now_sec, elapsed);
|
self.record_latency_atomic(now_sec, elapsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
|
|
||||||
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_call();
|
self.record_call();
|
||||||
self.record_latency(now_sec, elapsed);
|
self.record_latency(now_sec, elapsed);
|
||||||
@@ -779,7 +770,6 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Set disk as faulty
|
/// Set disk as faulty
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn set_faulty(&self) {
|
pub fn set_faulty(&self) {
|
||||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||||
}
|
}
|
||||||
@@ -860,7 +850,6 @@ impl DiskHealthTracker {
|
|||||||
became_offline
|
became_offline
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||||
let current = self.runtime_state();
|
let current = self.runtime_state();
|
||||||
if current == RuntimeDriveHealthState::Offline {
|
if current == RuntimeDriveHealthState::Offline {
|
||||||
@@ -991,13 +980,11 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get waiting operations count
|
/// Get waiting operations count
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn waiting_count(&self) -> u32 {
|
pub fn waiting_count(&self) -> u32 {
|
||||||
self.waiting.load(Ordering::Relaxed)
|
self.waiting.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get last success timestamp
|
/// Get last success timestamp
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn last_success(&self) -> i64 {
|
pub fn last_success(&self) -> i64 {
|
||||||
self.last_success.load(Ordering::Acquire)
|
self.last_success.load(Ordering::Acquire)
|
||||||
}
|
}
|
||||||
@@ -1039,6 +1026,21 @@ impl Default for DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Health check context key for tracking disk operations
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct HealthDiskCtxKey;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct HealthDiskCtxValue {
|
||||||
|
last_success: Arc<AtomicI64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealthDiskCtxValue {
|
||||||
|
fn log_success(&self) {
|
||||||
|
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -1070,6 +1072,10 @@ impl LocalDiskWrapper {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
|
||||||
|
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn new_with_reconnect_state(
|
pub(crate) fn new_with_reconnect_state(
|
||||||
disk: Arc<LocalDisk>,
|
disk: Arc<LocalDisk>,
|
||||||
health_check: bool,
|
health_check: bool,
|
||||||
@@ -1432,6 +1438,20 @@ impl LocalDiskWrapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
|
||||||
|
if want_id.is_none() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let stored_disk_id = self.disk.get_disk_id().await?;
|
||||||
|
|
||||||
|
if stored_disk_id != want_id {
|
||||||
|
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Check if disk ID is stale
|
/// Check if disk ID is stale
|
||||||
async fn check_disk_stale(&self) -> Result<()> {
|
async fn check_disk_stale(&self) -> Result<()> {
|
||||||
let Some(current_disk_id) = *self.disk_id.read().await else {
|
let Some(current_disk_id) = *self.disk_id.read().await else {
|
||||||
|
|||||||
@@ -48,7 +48,6 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
||||||
match io_err.kind() {
|
match io_err.kind() {
|
||||||
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
||||||
|
|||||||
@@ -178,7 +178,6 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||||
match fs::remove_file(path.as_ref()).await {
|
match fs::remove_file(path.as_ref()).await {
|
||||||
|
|||||||
@@ -665,7 +665,6 @@ async fn remove_empty_directory_tree_under_mount_lease(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
async fn remove_empty_directory_tree_with(
|
async fn remove_empty_directory_tree_with(
|
||||||
root: &Path,
|
root: &Path,
|
||||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||||
@@ -1017,29 +1016,13 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
|
|||||||
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
||||||
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
||||||
/// Default: false (uses page cache via mmap/pread).
|
/// Default: false (uses page cache via mmap/pread).
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
||||||
|
|
||||||
/// Minimum shard size threshold for O_DIRECT reads.
|
/// Minimum shard size threshold for O_DIRECT reads.
|
||||||
/// Only shards larger than this threshold will use O_DIRECT.
|
/// Only shards larger than this threshold will use O_DIRECT.
|
||||||
/// Default: 4MB.
|
/// Default: 4MB.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
||||||
@@ -1053,15 +1036,7 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
|||||||
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
||||||
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
||||||
/// Default: false (buffered writes via the page cache, as before).
|
/// Default: false (buffered writes via the page cache, as before).
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||||
@@ -1120,14 +1095,12 @@ macro_rules! cached_read_env {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT reads are enabled.
|
/// Check if O_DIRECT reads are enabled.
|
||||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
|
||||||
fn is_direct_io_read_enabled() -> bool =
|
fn is_direct_io_read_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT shard/part data writes are enabled.
|
/// Check if O_DIRECT shard/part data writes are enabled.
|
||||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
|
||||||
fn is_direct_io_write_enabled() -> bool =
|
fn is_direct_io_write_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
||||||
}
|
}
|
||||||
@@ -1483,7 +1456,6 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Get the O_DIRECT read threshold size.
|
/// Get the O_DIRECT read threshold size.
|
||||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
|
||||||
fn get_direct_io_read_threshold() -> usize =
|
fn get_direct_io_read_threshold() -> usize =
|
||||||
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
||||||
}
|
}
|
||||||
@@ -1701,20 +1673,12 @@ impl DirectIoWriteState {
|
|||||||
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
||||||
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
||||||
/// into one positioned write to keep the syscall count low.
|
/// into one positioned write to keep the syscall count low.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
||||||
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
||||||
/// flushed batch length, and every write offset stay alignment-correct.
|
/// flushed batch length, and every write offset stay alignment-correct.
|
||||||
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn direct_write_staging_capacity(align: usize) -> usize {
|
fn direct_write_staging_capacity(align: usize) -> usize {
|
||||||
debug_assert!(align.is_power_of_two() && align >= 512);
|
debug_assert!(align.is_power_of_two() && align >= 512);
|
||||||
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
||||||
@@ -1723,10 +1687,6 @@ fn direct_write_staging_capacity(align: usize) -> usize {
|
|||||||
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
||||||
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
||||||
/// so the tail-boundary math is unit-tested on any host.
|
/// so the tail-boundary math is unit-tested on any host.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
||||||
let aligned = filled - (filled % align);
|
let aligned = filled - (filled % align);
|
||||||
(aligned, filled - aligned)
|
(aligned, filled - aligned)
|
||||||
@@ -2182,7 +2142,6 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
||||||
DELETE_VERSION_FAIL_AFTER_COMMIT
|
DELETE_VERSION_FAIL_AFTER_COMMIT
|
||||||
.lock()
|
.lock()
|
||||||
@@ -2488,10 +2447,6 @@ enum SyncMode {
|
|||||||
FileOnly,
|
FileOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct FileCacheReclaimWriter {
|
struct FileCacheReclaimWriter {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_len: usize,
|
reclaim_len: usize,
|
||||||
@@ -2499,10 +2454,6 @@ struct FileCacheReclaimWriter {
|
|||||||
reclaimed: bool,
|
reclaimed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct FileCacheReclaimReader {
|
struct FileCacheReclaimReader {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_offset: u64,
|
reclaim_offset: u64,
|
||||||
@@ -2568,10 +2519,6 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||||
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
||||||
// (three metric-key constructions) when general metrics are disabled.
|
// (three metric-key constructions) when general metrics are disabled.
|
||||||
@@ -3124,7 +3071,6 @@ impl LocalIoBackend for StdBackend {
|
|||||||
use memmap2::MmapOptions;
|
use memmap2::MmapOptions;
|
||||||
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
||||||
|
|
||||||
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
|
|
||||||
struct MmapCopyReadResult {
|
struct MmapCopyReadResult {
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
access_check_duration: StdDuration,
|
access_check_duration: StdDuration,
|
||||||
@@ -4758,10 +4704,6 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
|||||||
Arc::new(StdBackend::new(root))
|
Arc::new(StdBackend::new(root))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub struct LocalDisk {
|
pub struct LocalDisk {
|
||||||
pub root: PathBuf,
|
pub root: PathBuf,
|
||||||
publication_root: os::PublicationRoot,
|
publication_root: os::PublicationRoot,
|
||||||
@@ -5548,7 +5490,6 @@ impl LocalDisk {
|
|||||||
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
||||||
let path_ref = path.as_ref();
|
let path_ref = path.as_ref();
|
||||||
let path_str = path_ref.to_string_lossy();
|
let path_str = path_ref.to_string_lossy();
|
||||||
@@ -5626,24 +5567,15 @@ impl LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check if a path is valid
|
// Check if a path is valid
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||||
check_local_disk_valid_path(self.io_root(), path)
|
check_local_disk_valid_path(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
||||||
reject_local_disk_symlink_components(self.io_root(), path)
|
reject_local_disk_symlink_components(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch path generation with single lock acquisition
|
// Batch path generation with single lock acquisition
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
||||||
let mut results = Vec::with_capacity(requests.len());
|
let mut results = Vec::with_capacity(requests.len());
|
||||||
let mut cache_misses = Vec::new();
|
let mut cache_misses = Vec::new();
|
||||||
@@ -6556,13 +6488,12 @@ impl LocalDisk {
|
|||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
fn get_metrics(&self) -> DiskMetrics {
|
fn get_metrics(&self) -> DiskMetrics {
|
||||||
DiskMetrics::default()
|
DiskMetrics::default()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
pub mod disk_store;
|
pub mod disk_store;
|
||||||
pub mod endpoint;
|
pub mod endpoint;
|
||||||
@@ -1113,10 +1114,6 @@ pub struct DiskInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub struct Info {
|
pub struct Info {
|
||||||
pub total: u64,
|
pub total: u64,
|
||||||
pub free: u64,
|
pub free: u64,
|
||||||
@@ -1375,7 +1372,6 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
||||||
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -571,10 +571,6 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
|||||||
|
|
||||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||||
/// itself.
|
/// itself.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
@@ -587,7 +583,6 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
|
|
||||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||||
/// concurrently, bounded both per directory and process-wide.
|
/// concurrently, bounded both per directory and process-wide.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
||||||
}
|
}
|
||||||
@@ -1814,6 +1809,10 @@ impl RenameCommitGuard {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
|
||||||
|
self.destination_directory_guard(directory, false)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn create_destination_directory_for_path_access(
|
pub(crate) fn create_destination_directory_for_path_access(
|
||||||
&self,
|
&self,
|
||||||
directory: &Path,
|
directory: &Path,
|
||||||
@@ -2859,6 +2858,13 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check if a file exists.
|
||||||
|
/// Returns true if the file exists, false otherwise.
|
||||||
|
#[tracing::instrument(level = "debug", skip_all)]
|
||||||
|
pub fn file_exists(path: impl AsRef<Path>) -> bool {
|
||||||
|
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether an [`io::Error`] means "the directory is not empty".
|
/// Whether an [`io::Error`] means "the directory is not empty".
|
||||||
///
|
///
|
||||||
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
||||||
|
|||||||
@@ -820,263 +820,10 @@ impl BitrotWriterWrapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
|
||||||
//
|
|
||||||
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
|
||||||
// key-handling regression) fails silently: every shard reads back "corrupt",
|
|
||||||
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
|
||||||
// which copy is healthy. The self-test below pins the algorithms the moment a
|
|
||||||
// process starts, so a drifted build announces itself instead of quietly
|
|
||||||
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
|
||||||
// analysis-2026-08-16.md §6 HS-11.
|
|
||||||
|
|
||||||
/// Length of the deterministic self-test payload.
|
|
||||||
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
|
||||||
|
|
||||||
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
|
||||||
/// (the production default). Pinned so any platform or build where the
|
|
||||||
/// implementation drifts fails startup instead of miss-hashing shards.
|
|
||||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
|
||||||
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
|
||||||
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
|
||||||
];
|
|
||||||
|
|
||||||
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
|
||||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
|
||||||
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
|
||||||
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
|
||||||
];
|
|
||||||
|
|
||||||
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
|
||||||
/// Highway digests above this one is externally verifiable, so it guards the
|
|
||||||
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
|
||||||
/// self-computed constants.
|
|
||||||
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
|
||||||
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
|
||||||
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
|
||||||
];
|
|
||||||
|
|
||||||
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
|
||||||
/// platform and every run hashes the same 4096 bytes.
|
|
||||||
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
|
||||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
|
||||||
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
|
||||||
for byte in payload.iter_mut() {
|
|
||||||
state ^= state >> 12;
|
|
||||||
state ^= state << 25;
|
|
||||||
state ^= state >> 27;
|
|
||||||
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
|
||||||
}
|
|
||||||
payload
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Why a bitrot self-test failed.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum BitrotSelfTestError {
|
|
||||||
/// A known-answer digest mismatched the pinned constant.
|
|
||||||
KnownAnswerMismatch {
|
|
||||||
algorithm: &'static str,
|
|
||||||
got: String,
|
|
||||||
want: String,
|
|
||||||
},
|
|
||||||
/// A freshly encoded shard failed `bitrot_verify`.
|
|
||||||
RoundtripVerify { algorithm: &'static str, detail: String },
|
|
||||||
/// A verified roundtrip read back different bytes than were written.
|
|
||||||
RoundtripReadback { algorithm: &'static str },
|
|
||||||
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
|
||||||
TamperNotRejected {
|
|
||||||
algorithm: &'static str,
|
|
||||||
tampered: &'static str,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for BitrotSelfTestError {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
match self {
|
|
||||||
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
|
||||||
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
|
||||||
}
|
|
||||||
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
|
||||||
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
|
||||||
Self::TamperNotRejected { algorithm, tampered } => {
|
|
||||||
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::error::Error for BitrotSelfTestError {}
|
|
||||||
|
|
||||||
fn self_test_hex(bytes: &[u8]) -> String {
|
|
||||||
rustfs_utils::hex(bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// (kept as a named one-liner so every KAT failure site reads the same; the
|
|
||||||
// underlying formatter is the shared `rustfs_utils::hex`)
|
|
||||||
|
|
||||||
/// Compare a digest against its pinned constant. Split out so a test can drive
|
|
||||||
/// it with a wrong constant and prove the mismatch path fires.
|
|
||||||
fn bitrot_kat_check(
|
|
||||||
algorithm: &'static str,
|
|
||||||
algo: &HashAlgorithm,
|
|
||||||
payload: &[u8],
|
|
||||||
expected: &[u8; 32],
|
|
||||||
) -> Result<(), BitrotSelfTestError> {
|
|
||||||
let digest = algo.hash_encode(payload);
|
|
||||||
let digest = digest.as_ref();
|
|
||||||
if digest.len() != expected.len() || digest != expected.as_slice() {
|
|
||||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
|
||||||
algorithm,
|
|
||||||
got: self_test_hex(digest),
|
|
||||||
want: self_test_hex(expected),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
|
||||||
/// every block back through `BitrotReader` comparing bytes.
|
|
||||||
async fn bitrot_roundtrip_check(
|
|
||||||
algorithm: &'static str,
|
|
||||||
algo: HashAlgorithm,
|
|
||||||
payload: &[u8],
|
|
||||||
shard_size: usize,
|
|
||||||
) -> Result<(), BitrotSelfTestError> {
|
|
||||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
|
||||||
for chunk in payload.chunks(shard_size) {
|
|
||||||
writer
|
|
||||||
.write(chunk)
|
|
||||||
.await
|
|
||||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
|
||||||
algorithm,
|
|
||||||
detail: format!("encode failed: {err}"),
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
let encoded = writer.into_inner().into_inner();
|
|
||||||
|
|
||||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
|
||||||
if encoded.len() != on_disk {
|
|
||||||
return Err(BitrotSelfTestError::RoundtripVerify {
|
|
||||||
algorithm,
|
|
||||||
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
|
||||||
.await
|
|
||||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
|
||||||
algorithm,
|
|
||||||
detail: err.to_string(),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
|
||||||
let mut offset = 0usize;
|
|
||||||
while offset < payload.len() {
|
|
||||||
let want = shard_size.min(payload.len() - offset);
|
|
||||||
let mut buf = vec![0u8; want];
|
|
||||||
let read = reader
|
|
||||||
.read(&mut buf)
|
|
||||||
.await
|
|
||||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
|
||||||
algorithm,
|
|
||||||
detail: format!("read back failed at offset {offset}: {err}"),
|
|
||||||
})?;
|
|
||||||
if read != want || buf[..read] != payload[offset..offset + read] {
|
|
||||||
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
|
||||||
}
|
|
||||||
offset += read;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Flip one byte and require `bitrot_verify` to reject the result.
|
|
||||||
async fn bitrot_tamper_check(
|
|
||||||
algorithm: &'static str,
|
|
||||||
algo: HashAlgorithm,
|
|
||||||
payload: &[u8],
|
|
||||||
shard_size: usize,
|
|
||||||
tampered: &'static str,
|
|
||||||
flip_at: usize,
|
|
||||||
) -> Result<(), BitrotSelfTestError> {
|
|
||||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
|
||||||
for chunk in payload.chunks(shard_size) {
|
|
||||||
writer.write(chunk).await.expect("self-test encode should not fail");
|
|
||||||
}
|
|
||||||
let mut corrupt = writer.into_inner().into_inner();
|
|
||||||
let flip_index = flip_at % corrupt.len();
|
|
||||||
corrupt[flip_index] ^= 0x80;
|
|
||||||
|
|
||||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
|
||||||
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
|
||||||
// The flipped byte must be rejected as a hash mismatch specifically, not
|
|
||||||
// by any incidental read error: an in-memory cursor cannot fail reads,
|
|
||||||
// so accepting any other failure here would mask a verify path that
|
|
||||||
// errors out before it ever compares hashes.
|
|
||||||
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
|
||||||
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
|
||||||
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
|
||||||
algorithm,
|
|
||||||
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Verify every bitrot algorithm this crate can write or verify in production:
|
|
||||||
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
|
||||||
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
|
||||||
/// and the leading hash, while all three hashed algorithms reproduce their
|
|
||||||
/// pinned known-answer digests.
|
|
||||||
///
|
|
||||||
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
|
||||||
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
|
||||||
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
|
||||||
let payload = bitrot_self_test_payload();
|
|
||||||
|
|
||||||
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
|
||||||
// itself, before any self-pinned constants are consulted.
|
|
||||||
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
|
||||||
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
|
||||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
|
||||||
algorithm: "SHA256",
|
|
||||||
got: self_test_hex(abc.as_ref()),
|
|
||||||
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
bitrot_kat_check(
|
|
||||||
"HighwayHash256S",
|
|
||||||
&HashAlgorithm::HighwayHash256S,
|
|
||||||
&payload,
|
|
||||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
|
||||||
)?;
|
|
||||||
bitrot_kat_check(
|
|
||||||
"HighwayHash256SLegacy",
|
|
||||||
&HashAlgorithm::HighwayHash256SLegacy,
|
|
||||||
&payload,
|
|
||||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
for (algorithm, algo) in [
|
|
||||||
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
|
||||||
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
|
||||||
] {
|
|
||||||
// Full blocks plus a partial tail, exactly like a real part stripe.
|
|
||||||
let tail_len = 2 * 1024 + 333;
|
|
||||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
|
||||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
|
||||||
// One flipped byte in the final data block, one in the first leading
|
|
||||||
// hash: both must fail verification.
|
|
||||||
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
|
||||||
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||||
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
|
||||||
};
|
};
|
||||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -1343,32 +1090,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn bitrot_self_test_payload_is_deterministic() {
|
|
||||||
// Two independent builds of the payload must agree byte for byte, or
|
|
||||||
// the pinned known-answer digests below would be meaningless.
|
|
||||||
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
|
||||||
let payload = bitrot_self_test_payload();
|
|
||||||
let wrong = [0u8; 32];
|
|
||||||
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
|
||||||
.expect_err("a zeroed digest must never match");
|
|
||||||
match err {
|
|
||||||
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
|
||||||
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn bitrot_self_test_passes() {
|
|
||||||
bitrot_self_test()
|
|
||||||
.await
|
|
||||||
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||||
let mut counting = VectoredCountingWriter::default();
|
let mut counting = VectoredCountingWriter::default();
|
||||||
@@ -1468,7 +1189,7 @@ mod tests {
|
|||||||
let last = corrupt.len() - 1;
|
let last = corrupt.len() - 1;
|
||||||
corrupt[last] ^= 0x80;
|
corrupt[last] ^= 0x80;
|
||||||
let err = bitrot_verify(
|
let err = bitrot_verify(
|
||||||
std::io::Cursor::new(corrupt),
|
Cursor::new(corrupt),
|
||||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||||
data.len(),
|
data.len(),
|
||||||
algo,
|
algo,
|
||||||
@@ -1561,7 +1282,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||||
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||||
let mut out = [0u8; 5];
|
let mut out = [0u8; 5];
|
||||||
let err = reader
|
let err = reader
|
||||||
.read(&mut out)
|
.read(&mut out)
|
||||||
@@ -1686,7 +1407,7 @@ mod tests {
|
|||||||
(HashAlgorithm::HighwayHash256, true),
|
(HashAlgorithm::HighwayHash256, true),
|
||||||
] {
|
] {
|
||||||
let label = format!("{algo:?}");
|
let label = format!("{algo:?}");
|
||||||
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
let writer = Cursor::new(Vec::<u8>::new());
|
||||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||||
w.write(&[7u8; 16]).await.unwrap();
|
w.write(&[7u8; 16]).await.unwrap();
|
||||||
let written = w.into_inner().into_inner();
|
let written = w.into_inner().into_inner();
|
||||||
@@ -1771,7 +1492,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||||
w.write(payload).await.unwrap();
|
w.write(payload).await.unwrap();
|
||||||
w.into_inner().into_inner()
|
w.into_inner().into_inner()
|
||||||
}
|
}
|
||||||
@@ -1879,7 +1600,7 @@ mod tests {
|
|||||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||||
for chunk in payload.chunks(shard_size) {
|
for chunk in payload.chunks(shard_size) {
|
||||||
w.write(chunk).await.unwrap();
|
w.write(chunk).await.unwrap();
|
||||||
}
|
}
|
||||||
@@ -1953,14 +1674,14 @@ mod tests {
|
|||||||
w.write(&data).await.expect("write shard");
|
w.write(&data).await.expect("write shard");
|
||||||
|
|
||||||
let mut via_read = vec![0u8; SHARD];
|
let mut via_read = vec![0u8; SHARD];
|
||||||
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||||
.read(&mut via_read)
|
.read(&mut via_read)
|
||||||
.await
|
.await
|
||||||
.expect("read");
|
.expect("read");
|
||||||
|
|
||||||
// A buffer with only capacity — no initialized bytes at all.
|
// A buffer with only capacity — no initialized bytes at all.
|
||||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||||
.read_appending(&mut via_append, SHARD)
|
.read_appending(&mut via_append, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect("read_appending");
|
.expect("read_appending");
|
||||||
@@ -1985,7 +1706,7 @@ mod tests {
|
|||||||
encoded.truncate(encoded.len() - 1);
|
encoded.truncate(encoded.len() - 1);
|
||||||
|
|
||||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||||
.read_appending(&mut out, SHARD)
|
.read_appending(&mut out, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect_err("a truncated shard must not succeed");
|
.expect_err("a truncated shard must not succeed");
|
||||||
@@ -2011,7 +1732,7 @@ mod tests {
|
|||||||
encoded[last] ^= 0xff;
|
encoded[last] ^= 0xff;
|
||||||
|
|
||||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||||
.read_appending(&mut out, SHARD)
|
.read_appending(&mut out, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect_err("a corrupt shard must not verify");
|
.expect_err("a corrupt shard must not verify");
|
||||||
@@ -2123,7 +1844,7 @@ mod tests {
|
|||||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||||
);
|
);
|
||||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||||
let mut streamed = std::io::Cursor::new(encoded.clone());
|
let mut streamed = Cursor::new(encoded.clone());
|
||||||
assert!(
|
assert!(
|
||||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||||
"a non-Bytes source must stay on the streaming path"
|
"a non-Bytes source must stay on the streaming path"
|
||||||
@@ -2151,7 +1872,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||||
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||||
.read_appending(&mut via_stream, SHARD)
|
.read_appending(&mut via_stream, SHARD)
|
||||||
.await
|
.await
|
||||||
.expect("streaming read");
|
.expect("streaming read");
|
||||||
|
|||||||
@@ -1116,14 +1116,6 @@ mod tests {
|
|||||||
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
||||||
}
|
}
|
||||||
|
|
||||||
// The lifecycle transition worker relies on this arm alone to suppress the
|
|
||||||
// closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would
|
|
||||||
// silently turn shutdown races back into `error!` log spam.
|
|
||||||
#[test]
|
|
||||||
fn is_network_or_host_down_covers_closed_network_connection() {
|
|
||||||
assert!(is_network_or_host_down("transition failed: use of closed network connection", false));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
||||||
// every set unreachable) must NOT be classified as "all not found",
|
// every set unreachable) must NOT be classified as "all not found",
|
||||||
// otherwise ListObjects silently returns an empty listing and masks a full
|
// otherwise ListObjects silently returns an empty listing and masks a full
|
||||||
|
|||||||
@@ -704,7 +704,6 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn create_deferred_bitrot_reader(
|
pub fn create_deferred_bitrot_reader(
|
||||||
inline_data: Option<Bytes>,
|
inline_data: Option<Bytes>,
|
||||||
disk: Option<DiskStore>,
|
disk: Option<DiskStore>,
|
||||||
|
|||||||
@@ -277,12 +277,6 @@ pub struct ObjectOptions {
|
|||||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||||
pub replication_request: bool,
|
pub replication_request: bool,
|
||||||
/// Source-cluster LWW timestamps carried by an authorized replication
|
|
||||||
/// request; None when the source never modified the category. Only the
|
|
||||||
/// replication-authorized options builders may set these.
|
|
||||||
pub replication_tagging_timestamp: Option<OffsetDateTime>,
|
|
||||||
pub replication_retention_timestamp: Option<OffsetDateTime>,
|
|
||||||
pub replication_legalhold_timestamp: Option<OffsetDateTime>,
|
|
||||||
/// Authorized SSE-C replication passthrough: the body is already
|
/// Authorized SSE-C replication passthrough: the body is already
|
||||||
/// ciphertext, so the write path must not encrypt or compress it and
|
/// ciphertext, so the write path must not encrypt or compress it and
|
||||||
/// stores the restored encryption metadata verbatim. Only the
|
/// stores the restored encryption metadata verbatim. Only the
|
||||||
|
|||||||
@@ -132,6 +132,7 @@ impl RebalanceStopPropagationRecord {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct DiskStat {
|
pub struct DiskStat {
|
||||||
pub total_space: u64,
|
pub total_space: u64,
|
||||||
|
|||||||
@@ -16,16 +16,8 @@ use serde::{Deserialize, Serialize};
|
|||||||
use std::{fmt::Display, io};
|
use std::{fmt::Display, io};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "tier config wire version stamped by the parity constructors below (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const C_TIER_CONFIG_VER: &str = "v1";
|
const C_TIER_CONFIG_VER: &str = "v1";
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||||
@@ -272,6 +264,7 @@ impl Clone for TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl TierConfig {
|
impl TierConfig {
|
||||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -291,7 +284,6 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn endpoint(&self) -> String {
|
fn endpoint(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||||
@@ -311,7 +303,6 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn bucket(&self) -> String {
|
fn bucket(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||||
@@ -331,7 +322,6 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn prefix(&self) -> String {
|
fn prefix(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||||
@@ -351,7 +341,6 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
|
||||||
fn region(&self) -> String {
|
fn region(&self) -> String {
|
||||||
match self.tier_type {
|
match self.tier_type {
|
||||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||||
@@ -468,7 +457,7 @@ impl TierWasabi {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TierS3 {
|
impl TierS3 {
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
fn create<F>(
|
fn create<F>(
|
||||||
name: &str,
|
name: &str,
|
||||||
access_key: &str,
|
access_key: &str,
|
||||||
@@ -539,7 +528,7 @@ pub struct TierMinIO {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TierMinIO {
|
impl TierMinIO {
|
||||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
#[allow(dead_code)]
|
||||||
fn create<F>(
|
fn create<F>(
|
||||||
name: &str,
|
name: &str,
|
||||||
endpoint: &str,
|
endpoint: &str,
|
||||||
|
|||||||
@@ -14,6 +14,7 @@
|
|||||||
|
|
||||||
use crate::services::tier::tier::TierConfigMgr;
|
use crate::services::tier::tier::TierConfigMgr;
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
impl TierConfigMgr {
|
impl TierConfigMgr {
|
||||||
pub fn msg_size(&self) -> usize {
|
pub fn msg_size(&self) -> usize {
|
||||||
100
|
100
|
||||||
|
|||||||
@@ -32,22 +32,15 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
||||||
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
||||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK,
|
||||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
|
||||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
|
||||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
|
||||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
|
||||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
|
||||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
|
||||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||||
@@ -180,13 +173,11 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
|
|||||||
Fallback(GetCodecStreamingFallbackReason),
|
Fallback(GetCodecStreamingFallbackReason),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
||||||
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MultipartCodecStreamingReader {
|
impl MultipartCodecStreamingReader {
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
readers: VecDeque::from(readers),
|
readers: VecDeque::from(readers),
|
||||||
@@ -661,15 +652,36 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
|||||||
&& left.erasure.distribution == right.erasure.distribution
|
&& left.erasure.distribution == right.erasure.distribution
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
candidate: &FileInfo,
|
candidate: &FileInfo,
|
||||||
parts_metadata: &[FileInfo],
|
parts_metadata: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
) -> Option<&'static str> {
|
) -> bool {
|
||||||
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
|
if !candidate.inline_data()
|
||||||
return Some(reason);
|
|| candidate.is_compressed()
|
||||||
|
|| candidate
|
||||||
|
.metadata
|
||||||
|
.keys()
|
||||||
|
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||||
|
|| candidate.is_remote()
|
||||||
|
|| candidate.deleted
|
||||||
|
|| candidate.size <= 0
|
||||||
|
|| candidate.parts.len() != 1
|
||||||
|
|| !candidate.has_valid_erasure_geometry()
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||||
@@ -678,21 +690,18 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
|||||||
candidate.erasure.block_size,
|
candidate.erasure.block_size,
|
||||||
candidate.uses_legacy_checksum,
|
candidate.uses_legacy_checksum,
|
||||||
) else {
|
) else {
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
return false;
|
||||||
};
|
};
|
||||||
let data_files =
|
let Some(data_files) =
|
||||||
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
|
collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| {
|
||||||
disks.get(index).is_some_and(Option::is_some)
|
disks.get(index).is_some_and(Option::is_some)
|
||||||
}) {
|
})
|
||||||
Ok(data_files) => data_files,
|
else {
|
||||||
Err(reason) => return Some(reason),
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(part) = candidate.parts.first() else {
|
let Some(part) = candidate.parts.first() else {
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
return false;
|
||||||
};
|
|
||||||
let Ok(object_size) = usize::try_from(candidate.size) else {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
|
||||||
};
|
};
|
||||||
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
||||||
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||||
@@ -712,111 +721,12 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
|||||||
let Ok(mut readers) =
|
let Ok(mut readers) =
|
||||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||||
else {
|
else {
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
|
||||||
};
|
|
||||||
|
|
||||||
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
|
||||||
Some(body) if body.len() == object_size => None,
|
|
||||||
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
|
||||||
// `inline_data` excludes remote objects; this diagnostic reports them separately.
|
|
||||||
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
|
|
||||||
}
|
|
||||||
if candidate.is_compressed()
|
|
||||||
|| candidate
|
|
||||||
.metadata
|
|
||||||
.keys()
|
|
||||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
|
||||||
{
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED);
|
|
||||||
}
|
|
||||||
if candidate.is_remote() {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
|
|
||||||
}
|
|
||||||
if candidate.deleted {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
|
|
||||||
}
|
|
||||||
if candidate.size <= 0 {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
|
||||||
}
|
|
||||||
if candidate.parts.len() != 1 {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
|
||||||
}
|
|
||||||
if !candidate.has_valid_erasure_geometry() {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
|
||||||
}
|
|
||||||
|
|
||||||
let Ok(object_size) = usize::try_from(candidate.size) else {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
|
||||||
};
|
|
||||||
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
|
||||||
}
|
|
||||||
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
|
||||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn data_read_inline_missing_shards_are_pending(
|
|
||||||
candidate: &FileInfo,
|
|
||||||
parts_metadata: &[FileInfo],
|
|
||||||
errors: &[Option<DiskError>],
|
|
||||||
disks: &[Option<DiskStore>],
|
|
||||||
fanout_order: &[usize],
|
|
||||||
scheduled_fanout_len: usize,
|
|
||||||
) -> bool {
|
|
||||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
|
||||||
candidate.erasure.data_blocks,
|
|
||||||
candidate.erasure.parity_blocks,
|
|
||||||
candidate.erasure.block_size,
|
|
||||||
candidate.uses_legacy_checksum,
|
|
||||||
) else {
|
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let distribution = &candidate.erasure.distribution;
|
|
||||||
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
|
|
||||||
let mut missing_pending_data_shards = 0usize;
|
|
||||||
|
|
||||||
for (disk_index, file_info) in parts_metadata.iter().enumerate() {
|
try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size)
|
||||||
let Some(&block_index) = distribution.get(disk_index) else {
|
.await
|
||||||
return false;
|
.is_some_and(|body| body.len() == object_size)
|
||||||
};
|
|
||||||
if block_index == 0 || block_index > erasure.data_shards {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if !disks.get(disk_index).is_some_and(Option::is_some) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
let data_slot = block_index - 1;
|
|
||||||
if file_info.name.is_empty() {
|
|
||||||
let scheduled_and_not_failed = fanout_order
|
|
||||||
.get(..scheduled_fanout_len)
|
|
||||||
.is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index))
|
|
||||||
&& errors.get(disk_index).is_some_and(Option::is_none);
|
|
||||||
if scheduled_and_not_failed {
|
|
||||||
data_shards_seen_or_pending[data_slot] = true;
|
|
||||||
missing_pending_data_shards = missing_pending_data_shards.saturating_add(1);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if file_info.erasure.index != block_index
|
|
||||||
|| !file_info.has_valid_erasure_geometry()
|
|
||||||
|| !metadata_early_stop_candidate_matches(file_info, candidate)
|
|
||||||
|| file_info.data.as_ref().is_none_or(|data| data.is_empty())
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
data_shards_seen_or_pending[data_slot] = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
||||||
@@ -1848,7 +1758,6 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
||||||
files: &[FileInfo],
|
files: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
@@ -2139,7 +2048,6 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
|
|||||||
setup
|
setup
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
||||||
tasks: Vec<F>,
|
tasks: Vec<F>,
|
||||||
read_quorum: usize,
|
read_quorum: usize,
|
||||||
@@ -2456,7 +2364,6 @@ impl SetDisks {
|
|||||||
let bucket: Arc<str> = Arc::from(bucket);
|
let bucket: Arc<str> = Arc::from(bucket);
|
||||||
let object: Arc<str> = Arc::from(object);
|
let object: Arc<str> = Arc::from(object);
|
||||||
let version_id: Arc<str> = Arc::from(version_id);
|
let version_id: Arc<str> = Arc::from(version_id);
|
||||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
|
||||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||||
let disk = disk.clone();
|
let disk = disk.clone();
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2464,14 +2371,10 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
let slowtail_fault = slowtail_fault.clone();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let response_start = observe.then(Instant::now);
|
let response_start = observe.then(Instant::now);
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
Self::record_read_version_call(&object, disk_index);
|
Self::record_read_version_call(&object, disk_index);
|
||||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
|
||||||
tokio::time::sleep(delay).await;
|
|
||||||
}
|
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2566,8 +2469,6 @@ impl SetDisks {
|
|||||||
let mut next_fanout_index = 0usize;
|
let mut next_fanout_index = 0usize;
|
||||||
let mut scheduled_count = 0usize;
|
let mut scheduled_count = 0usize;
|
||||||
let mut force_full_wait = false;
|
let mut force_full_wait = false;
|
||||||
let mut final_miss_reason_override = None;
|
|
||||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
|
||||||
let spawn_read_version =
|
let spawn_read_version =
|
||||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2575,7 +2476,6 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
let slowtail_fault = slowtail_fault.clone();
|
|
||||||
join_set.spawn(async move {
|
join_set.spawn(async move {
|
||||||
let response_start = Instant::now();
|
let response_start = Instant::now();
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
@@ -2584,9 +2484,6 @@ impl SetDisks {
|
|||||||
Self::record_read_version_call(&object, index);
|
Self::record_read_version_call(&object, index);
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
Self::read_version_fanout_barrier(&object, index).await;
|
Self::read_version_fanout_barrier(&object, index).await;
|
||||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
|
||||||
tokio::time::sleep(delay).await;
|
|
||||||
}
|
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2614,20 +2511,11 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
while let Some(result) = join_set.join_next().await {
|
while let Some(result) = join_set.join_next().await {
|
||||||
let mut defer_pending_inline_data_shard = false;
|
|
||||||
match result {
|
match result {
|
||||||
Ok((index, res, elapsed)) => match res {
|
Ok((index, res, elapsed)) => match res {
|
||||||
Ok(file_info) => {
|
Ok(file_info) => {
|
||||||
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
||||||
accumulator.observe_file_info(&file_info);
|
accumulator.observe_file_info(&file_info);
|
||||||
if bounded_fanout
|
|
||||||
&& read_data
|
|
||||||
&& !force_full_wait
|
|
||||||
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
|
|
||||||
{
|
|
||||||
force_full_wait = true;
|
|
||||||
final_miss_reason_override.get_or_insert(reason);
|
|
||||||
}
|
|
||||||
if let Some(slot) = ress.get_mut(index) {
|
if let Some(slot) = ress.get_mut(index) {
|
||||||
*slot = file_info;
|
*slot = file_info;
|
||||||
}
|
}
|
||||||
@@ -2653,43 +2541,17 @@ impl SetDisks {
|
|||||||
.or_else(|| accumulator.version_early_stop_decision())
|
.or_else(|| accumulator.version_early_stop_decision())
|
||||||
{
|
{
|
||||||
let should_return_early = if read_data {
|
let should_return_early = if read_data {
|
||||||
match accumulator.candidate.as_ref() {
|
let allow_data_read_early_stop = match accumulator.candidate.as_ref() {
|
||||||
Some(candidate) => match data_read_early_stop_inline_body_miss_reason(
|
Some(candidate) => {
|
||||||
bucket.as_ref(),
|
data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks)
|
||||||
object.as_ref(),
|
.await
|
||||||
candidate,
|
|
||||||
&ress,
|
|
||||||
disks,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
None => true,
|
|
||||||
Some(reason) => {
|
|
||||||
final_miss_reason_override = Some(reason);
|
|
||||||
if bounded_fanout
|
|
||||||
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
|
|
||||||
&& data_read_inline_missing_shards_are_pending(
|
|
||||||
candidate,
|
|
||||||
&ress,
|
|
||||||
&errors,
|
|
||||||
disks,
|
|
||||||
&fanout_order,
|
|
||||||
next_fanout_index,
|
|
||||||
)
|
|
||||||
{
|
|
||||||
defer_pending_inline_data_shard = true;
|
|
||||||
} else {
|
|
||||||
force_full_wait = true;
|
|
||||||
}
|
|
||||||
false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
None => {
|
|
||||||
force_full_wait = true;
|
|
||||||
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
|
|
||||||
false
|
|
||||||
}
|
}
|
||||||
|
None => false,
|
||||||
|
};
|
||||||
|
if !allow_data_read_early_stop {
|
||||||
|
force_full_wait = true;
|
||||||
}
|
}
|
||||||
|
allow_data_read_early_stop
|
||||||
} else {
|
} else {
|
||||||
true
|
true
|
||||||
};
|
};
|
||||||
@@ -2726,7 +2588,6 @@ impl SetDisks {
|
|||||||
let pending_responses = join_set.len();
|
let pending_responses = join_set.len();
|
||||||
let should_hedge_single_pending_data_read = read_data
|
let should_hedge_single_pending_data_read = read_data
|
||||||
&& !force_full_wait
|
&& !force_full_wait
|
||||||
&& !defer_pending_inline_data_shard
|
|
||||||
&& pending_responses == 1
|
&& pending_responses == 1
|
||||||
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
||||||
if bounded_fanout && force_full_wait {
|
if bounded_fanout && force_full_wait {
|
||||||
@@ -2739,7 +2600,6 @@ impl SetDisks {
|
|||||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||||
}
|
}
|
||||||
} else if bounded_fanout
|
} else if bounded_fanout
|
||||||
&& !defer_pending_inline_data_shard
|
|
||||||
&& next_fanout_index < disks.len()
|
&& next_fanout_index < disks.len()
|
||||||
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||||
|| should_hedge_single_pending_data_read)
|
|| should_hedge_single_pending_data_read)
|
||||||
@@ -2753,12 +2613,7 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let accumulator_miss_reason = accumulator.final_miss_reason();
|
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason());
|
||||||
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
|
|
||||||
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
|
|
||||||
_ => accumulator_miss_reason,
|
|
||||||
};
|
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
|
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
||||||
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
||||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||||
@@ -2987,7 +2842,6 @@ impl SetDisks {
|
|||||||
(meta_file_infos, errs)
|
(meta_file_infos, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) async fn read_multiple_files(
|
pub(in crate::set_disk) async fn read_multiple_files(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
req: ReadMultipleReq,
|
req: ReadMultipleReq,
|
||||||
@@ -3021,11 +2875,14 @@ impl SetDisks {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||||
Ok(collected) => collected,
|
Ok(collected) => collected,
|
||||||
Err(()) => return empty_quorum_result(),
|
Err(()) => return empty_quorum_result(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// debug!("ReadMultipleResp ress {:?}", ress);
|
||||||
|
// debug!("ReadMultipleResp errors {:?}", errors);
|
||||||
|
|
||||||
let mut ret = Vec::with_capacity(req.files.len());
|
let mut ret = Vec::with_capacity(req.files.len());
|
||||||
|
|
||||||
for want in req.files.iter() {
|
for want in req.files.iter() {
|
||||||
@@ -3164,7 +3021,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
|||||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
type RenameDataLegacyTuple = (
|
type RenameDataLegacyTuple = (
|
||||||
Vec<Option<DiskStore>>,
|
Vec<Option<DiskStore>>,
|
||||||
RenameConvergence,
|
RenameConvergence,
|
||||||
@@ -3174,7 +3030,6 @@ type RenameDataLegacyTuple = (
|
|||||||
);
|
);
|
||||||
|
|
||||||
impl RenameDataCommit {
|
impl RenameDataCommit {
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
||||||
(
|
(
|
||||||
self.online_disks,
|
self.online_disks,
|
||||||
@@ -3293,7 +3148,6 @@ impl SetDisks {
|
|||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||||
#[allow(clippy::type_complexity)]
|
#[allow(clippy::type_complexity)]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) async fn rename_data(
|
pub(in crate::set_disk) async fn rename_data(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
@@ -4860,14 +4714,6 @@ impl SetDisks {
|
|||||||
/// is best-effort maintenance: individual delete failures are logged and
|
/// is best-effort maintenance: individual delete failures are logged and
|
||||||
/// skipped rather than propagated.
|
/// skipped rather than propagated.
|
||||||
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||||
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
|
||||||
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
|
||||||
let disks = self.get_disks_internal().await;
|
let disks = self.get_disks_internal().await;
|
||||||
|
|
||||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||||
@@ -4975,20 +4821,6 @@ impl SetDisks {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let stray = format!("{object}/{dir}");
|
let stray = format!("{object}/{dir}");
|
||||||
if dry_run {
|
|
||||||
removed += 1;
|
|
||||||
debug!(
|
|
||||||
target: "rustfs_ecstore::set_disk",
|
|
||||||
event = "heal_abandoned_parts",
|
|
||||||
component = "ecstore",
|
|
||||||
subsystem = "heal",
|
|
||||||
state = "dry_run_matched",
|
|
||||||
result = "matched",
|
|
||||||
bucket, object, data_dir = %dir,
|
|
||||||
"Heal abandoned parts dry-run matched orphaned data directory"
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
match disk
|
match disk
|
||||||
.delete(
|
.delete(
|
||||||
bucket,
|
bucket,
|
||||||
@@ -5128,7 +4960,6 @@ fn is_cleanup_not_found(e: &DiskError) -> bool {
|
|||||||
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
||||||
/// must not be silently swallowed as an ignorable error (fixes the historical
|
/// must not be silently swallowed as an ignorable error (fixes the historical
|
||||||
/// `Unexpected`/`DiskNotFound` misclassification).
|
/// `Unexpected`/`DiskNotFound` misclassification).
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
||||||
match joined {
|
match joined {
|
||||||
Ok(res) => res,
|
Ok(res) => res,
|
||||||
@@ -5353,7 +5184,6 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
|||||||
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
||||||
pub const CLEANUP: &str = "cleanup";
|
pub const CLEANUP: &str = "cleanup";
|
||||||
/// The per-disk `read_version` phase of metadata read fan-out.
|
/// The per-disk `read_version` phase of metadata read fan-out.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub const READ_VERSION: &str = "read_version";
|
pub const READ_VERSION: &str = "read_version";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5791,130 +5621,6 @@ mod tests {
|
|||||||
(dirs, disks)
|
(dirs, disks)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
|
|
||||||
temp_env::with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
|
||||||
],
|
|
||||||
|| {
|
|
||||||
assert_eq!(
|
|
||||||
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
|
|
||||||
Some(Duration::from_millis(25))
|
|
||||||
);
|
|
||||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
|
|
||||||
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
|
|
||||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
|
|
||||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
|
|
||||||
temp_env::with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
|
|
||||||
],
|
|
||||||
|| {
|
|
||||||
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
||||||
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
|
|
||||||
const DISKS: usize = 4;
|
|
||||||
let bucket = "metadata-slowtail-fault-bucket";
|
|
||||||
let object = "objects/metadata-slowtail-fault-object";
|
|
||||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
|
||||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
|
||||||
|
|
||||||
temp_env::async_with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
|
||||||
],
|
|
||||||
async {
|
|
||||||
let read_without_data =
|
|
||||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
|
|
||||||
tokio::time::timeout(Duration::from_millis(100), read_without_data)
|
|
||||||
.await
|
|
||||||
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
|
|
||||||
.expect("metadata fanout without read_data should resolve");
|
|
||||||
|
|
||||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
|
||||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
|
||||||
));
|
|
||||||
assert!(
|
|
||||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
|
||||||
.await
|
|
||||||
.is_err(),
|
|
||||||
"data-read metadata fanout must wait for the injected slow read_version response"
|
|
||||||
);
|
|
||||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
|
||||||
.await
|
|
||||||
.expect("injected slowtail should eventually complete")
|
|
||||||
.expect("data-read metadata fanout should resolve");
|
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
|
||||||
assert!(errs.iter().all(Option::is_none));
|
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
drop(dirs);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
||||||
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
|
|
||||||
const DISKS: usize = 4;
|
|
||||||
let bucket = "metadata-slowtail-early-stop-bucket";
|
|
||||||
let object = "objects/metadata-slowtail-early-stop-object";
|
|
||||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
|
||||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
|
||||||
|
|
||||||
temp_env::async_with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
|
||||||
],
|
|
||||||
async {
|
|
||||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
|
||||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
|
||||||
));
|
|
||||||
assert!(
|
|
||||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
|
||||||
.await
|
|
||||||
.is_err(),
|
|
||||||
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
|
|
||||||
);
|
|
||||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
|
||||||
.await
|
|
||||||
.expect("injected early-stop slowtail should eventually complete")
|
|
||||||
.expect("early-stop metadata fanout should resolve");
|
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
|
||||||
assert!(errs.iter().all(Option::is_none));
|
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
drop(dirs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||||
///
|
///
|
||||||
/// The metadata fan-out issues each `read_version` inside its own
|
/// The metadata fan-out issues each `read_version` inside its own
|
||||||
@@ -6046,20 +5752,9 @@ mod tests {
|
|||||||
object: &str,
|
object: &str,
|
||||||
payload: &[u8],
|
payload: &[u8],
|
||||||
uses_legacy_checksum: bool,
|
uses_legacy_checksum: bool,
|
||||||
) -> Vec<FileInfo> {
|
|
||||||
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn inline_metadata_fanout_fileinfos_with_geometry(
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
payload: &[u8],
|
|
||||||
uses_legacy_checksum: bool,
|
|
||||||
data_shards: usize,
|
|
||||||
parity_shards: usize,
|
|
||||||
) -> Vec<FileInfo> {
|
) -> Vec<FileInfo> {
|
||||||
let distribution_key = metadata_distribution_key(bucket, object);
|
let distribution_key = metadata_distribution_key(bucket, object);
|
||||||
let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards);
|
let mut base = FileInfo::new(&distribution_key, 2, 2);
|
||||||
base.volume = bucket.to_string();
|
base.volume = bucket.to_string();
|
||||||
base.name = object.to_string();
|
base.name = object.to_string();
|
||||||
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
||||||
@@ -6122,21 +5817,6 @@ mod tests {
|
|||||||
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn install_inline_metadata_fanout_fileinfo_with_geometry(
|
|
||||||
disks: &[Option<DiskStore>],
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
payload: &[u8],
|
|
||||||
data_shards: usize,
|
|
||||||
parity_shards: usize,
|
|
||||||
mutate: impl FnOnce(&mut [FileInfo]),
|
|
||||||
) {
|
|
||||||
let mut files =
|
|
||||||
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await;
|
|
||||||
mutate(&mut files);
|
|
||||||
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
||||||
let distribution = files
|
let distribution = files
|
||||||
.first()
|
.first()
|
||||||
@@ -6357,118 +6037,6 @@ mod tests {
|
|||||||
drop(dirs);
|
drop(dirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
||||||
async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() {
|
|
||||||
const DISKS: usize = 6;
|
|
||||||
const DATA_SHARDS: usize = 4;
|
|
||||||
const PARITY_SHARDS: usize = 2;
|
|
||||||
let bucket = "bounded-inline-data-get-pending-shard-bucket";
|
|
||||||
let object =
|
|
||||||
object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS);
|
|
||||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
|
||||||
install_inline_metadata_fanout_fileinfo_with_geometry(
|
|
||||||
&disks,
|
|
||||||
bucket,
|
|
||||||
&object,
|
|
||||||
b"verified inline payload",
|
|
||||||
DATA_SHARDS,
|
|
||||||
PARITY_SHARDS,
|
|
||||||
|_| {},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
temp_env::async_with_vars(
|
|
||||||
[
|
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
|
||||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
|
||||||
],
|
|
||||||
async {
|
|
||||||
let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS);
|
|
||||||
let distribution_key = metadata_distribution_key(bucket, &object);
|
|
||||||
let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS)
|
|
||||||
.erasure
|
|
||||||
.distribution;
|
|
||||||
let paused_data_disk = *fanout_order
|
|
||||||
.iter()
|
|
||||||
.take(DATA_SHARDS)
|
|
||||||
.find(|disk_index| {
|
|
||||||
distribution
|
|
||||||
.get(**disk_index)
|
|
||||||
.is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index))
|
|
||||||
})
|
|
||||||
.expect("initial fanout should include a data shard to pause");
|
|
||||||
let hedged_parity_disk = fanout_order[DATA_SHARDS];
|
|
||||||
let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1];
|
|
||||||
|
|
||||||
let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
|
||||||
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
|
||||||
let calls = disk_call_counters::observe(&object);
|
|
||||||
let disks_for_read = disks.clone();
|
|
||||||
let object_for_read = object.clone();
|
|
||||||
let mut read = tokio::spawn(async move {
|
|
||||||
SetDisks::read_all_fileinfo_observed(
|
|
||||||
&disks_for_read,
|
|
||||||
bucket,
|
|
||||||
bucket,
|
|
||||||
&object_for_read,
|
|
||||||
"",
|
|
||||||
true,
|
|
||||||
false,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
PARITY_SHARDS,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
});
|
|
||||||
|
|
||||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
|
||||||
.await
|
|
||||||
.expect("initial data shard should pause before returning");
|
|
||||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
|
||||||
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 {
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("bounded fanout should hedge one parity disk while the data shard is pending");
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
|
|
||||||
"inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait"
|
|
||||||
);
|
|
||||||
|
|
||||||
barrier.release();
|
|
||||||
let (parts_metadata, errs, diagnostics) = read
|
|
||||||
.await
|
|
||||||
.expect("metadata read task should not panic")
|
|
||||||
.expect("pending data shard should let the inline verifier finish");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
|
||||||
5,
|
|
||||||
"pending data-shard defer should not schedule the final parity disk"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk),
|
|
||||||
0,
|
|
||||||
"the remaining parity disk must stay unissued when pending data verification succeeds"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
tracker.running(),
|
|
||||||
0,
|
|
||||||
"early-stop should drain spawned read_version tasks before returning"
|
|
||||||
);
|
|
||||||
assert_eq!(diagnostics.total_responses(), 5);
|
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5);
|
|
||||||
assert!(errs.iter().all(Option::is_none));
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
drop(dirs);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
||||||
let bucket = "legacy-inline-data-get-fanout-bucket";
|
let bucket = "legacy-inline-data-get-fanout-bucket";
|
||||||
@@ -6499,133 +6067,11 @@ mod tests {
|
|||||||
.clone();
|
.clone();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks)
|
data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await,
|
||||||
.await
|
|
||||||
.is_none(),
|
|
||||||
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn data_read_early_stop_reports_inline_miss_reasons() {
|
|
||||||
let bucket = "inline-data-get-miss-reason-bucket";
|
|
||||||
let object = "inline-data-get-miss-reason-object";
|
|
||||||
let payload = b"verified inline payload";
|
|
||||||
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
|
|
||||||
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
|
|
||||||
let distribution = files
|
|
||||||
.first()
|
|
||||||
.map(|file| file.erasure.distribution.clone())
|
|
||||||
.expect("fixture should include metadata");
|
|
||||||
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
|
|
||||||
let mut parts_metadata = vec![FileInfo::default(); 4];
|
|
||||||
for disk_index in order.into_iter().take(3) {
|
|
||||||
let block_index = distribution
|
|
||||||
.get(disk_index)
|
|
||||||
.copied()
|
|
||||||
.expect("fixture distribution should cover every disk");
|
|
||||||
parts_metadata[disk_index] = files
|
|
||||||
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
|
|
||||||
.expect("fixture should include every distributed shard")
|
|
||||||
.clone();
|
|
||||||
}
|
|
||||||
let candidate = parts_metadata
|
|
||||||
.iter()
|
|
||||||
.find(|file| file.name == object)
|
|
||||||
.expect("fixture should include observed metadata")
|
|
||||||
.clone();
|
|
||||||
let data_disk = distribution
|
|
||||||
.iter()
|
|
||||||
.position(|block_index| *block_index == 1)
|
|
||||||
.expect("fixture distribution should include first data shard");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
|
|
||||||
None
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut not_inline = candidate.clone();
|
|
||||||
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, ¬_inline, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut remote = candidate.clone();
|
|
||||||
remote.transition_status = TRANSITION_COMPLETE.to_string();
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut transformed = candidate.clone();
|
|
||||||
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut deleted = candidate.clone();
|
|
||||||
deleted.deleted = true;
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut zero_size = candidate.clone();
|
|
||||||
zero_size.size = 0;
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut multipart = candidate.clone();
|
|
||||||
multipart.parts.push(multipart.parts[0].clone());
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut invalid_geometry = candidate.clone();
|
|
||||||
invalid_geometry.erasure.data_blocks = 0;
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut missing_shard = parts_metadata.clone();
|
|
||||||
missing_shard[data_disk] = FileInfo::default();
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut missing_payload = parts_metadata.clone();
|
|
||||||
missing_payload[data_disk].data = None;
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut identity_mismatch = parts_metadata.clone();
|
|
||||||
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut corrupt = parts_metadata.clone();
|
|
||||||
if let Some(data) = corrupt[data_disk].data.as_mut() {
|
|
||||||
let mut corrupt_data = data.to_vec();
|
|
||||||
corrupt_data[0] ^= 0x01;
|
|
||||||
*data = Bytes::from(corrupt_data);
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
|
|
||||||
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
||||||
@@ -6715,7 +6161,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_INTERNAL_META),
|
("path", GET_OBJECT_PATH_INTERNAL_META),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
1,
|
1,
|
||||||
@@ -6727,7 +6173,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
0,
|
0,
|
||||||
@@ -7262,7 +6708,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() {
|
async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-hedge-bucket";
|
let bucket = "bounded-data-get-hedge-bucket";
|
||||||
let object = "bounded-data-get-hedge-object";
|
let object = "bounded-data-get-hedge-object";
|
||||||
@@ -7293,7 +6739,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk");
|
.expect("bounded data-read fanout should hedge by starting the spare disk");
|
||||||
|
|
||||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||||
assert!(
|
assert!(
|
||||||
@@ -7309,7 +6755,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare"
|
"bounded data-read fanout should issue the paused disk plus one spare hedge"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
@@ -7322,7 +6768,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() {
|
async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-default-bucket";
|
let bucket = "bounded-data-get-default-bucket";
|
||||||
let object = "bounded-data-get-default-object";
|
let object = "bounded-data-get-default-object";
|
||||||
@@ -7336,42 +6782,16 @@ mod tests {
|
|||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
|
|
||||||
let calls = disk_call_counters::observe(object);
|
let calls = disk_call_counters::observe(object);
|
||||||
let disks_for_read = disks.clone();
|
let (parts_metadata, errs, diagnostics) =
|
||||||
let mut read = tokio::spawn(async move {
|
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
|
||||||
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
|
|
||||||
.await
|
.await
|
||||||
});
|
.expect("default data-read metadata should resolve");
|
||||||
|
|
||||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
|
||||||
.await
|
|
||||||
.expect("default bounded non-inline read should schedule the paused metadata task");
|
|
||||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
|
||||||
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect(
|
|
||||||
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
|
|
||||||
);
|
|
||||||
|
|
||||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
|
||||||
assert!(
|
|
||||||
pending.is_err(),
|
|
||||||
"default non-inline data reads must not return before the paused metadata response"
|
|
||||||
);
|
|
||||||
barrier.release();
|
|
||||||
let (parts_metadata, errs, diagnostics) = read
|
|
||||||
.await
|
|
||||||
.expect("metadata read task should not panic")
|
|
||||||
.expect("default data-read metadata should resolve");
|
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first"
|
"default GET data-read metadata must keep full fanout for read-failure tolerance"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
|||||||
@@ -42,20 +42,12 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The borrowed core, for state not yet fronted by a typed accessor.
|
/// The borrowed core, for state not yet fronted by a typed accessor.
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn core(&self) -> &'a SetDisks {
|
pub(crate) fn core(&self) -> &'a SetDisks {
|
||||||
self.core
|
self.core
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Immutable topology / config (fixed after construction) ---
|
// --- Immutable topology / config (fixed after construction) ---
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn set_index(&self) -> usize {
|
pub(crate) fn set_index(&self) -> usize {
|
||||||
self.core.set_index
|
self.core.set_index
|
||||||
}
|
}
|
||||||
@@ -64,26 +56,14 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
self.core.pool_index
|
self.core.pool_index
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn set_drive_count(&self) -> usize {
|
pub(crate) fn set_drive_count(&self) -> usize {
|
||||||
self.core.set_drive_count
|
self.core.set_drive_count
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn default_parity_count(&self) -> usize {
|
pub(crate) fn default_parity_count(&self) -> usize {
|
||||||
self.core.default_parity_count
|
self.core.default_parity_count
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
||||||
&self.core.set_endpoints
|
&self.core.set_endpoints
|
||||||
}
|
}
|
||||||
@@ -92,10 +72,6 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
&self.core.format
|
&self.core.format
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn locker_owner(&self) -> &'a str {
|
pub(crate) fn locker_owner(&self) -> &'a str {
|
||||||
&self.core.locker_owner
|
&self.core.locker_owner
|
||||||
}
|
}
|
||||||
@@ -108,10 +84,6 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
|
|
||||||
// --- Locker trio ---
|
// --- Locker trio ---
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
||||||
&self.core.lockers
|
&self.core.lockers
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,6 +39,7 @@
|
|||||||
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
||||||
|
|
||||||
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
||||||
|
#![allow(dead_code)]
|
||||||
#![allow(unused_imports)]
|
#![allow(unused_imports)]
|
||||||
#![allow(unused_variables)]
|
#![allow(unused_variables)]
|
||||||
|
|
||||||
@@ -58,10 +59,7 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
|||||||
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
||||||
use crate::data_usage::record_compression_total_memory;
|
use crate::data_usage::record_compression_total_memory;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
|
|
||||||
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
|
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||||
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
|
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||||
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
|
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
|
||||||
@@ -102,7 +100,9 @@ use crate::storage_api_contracts::{
|
|||||||
};
|
};
|
||||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||||
use crate::{
|
use crate::{
|
||||||
bucket::lifecycle::bucket_lifecycle_ops::{LifecycleOps, get_transitioned_object_reader_with_tier_manager, put_restore_opts},
|
bucket::lifecycle::bucket_lifecycle_ops::{
|
||||||
|
LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts,
|
||||||
|
},
|
||||||
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
|
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
|
||||||
config::storageclass,
|
config::storageclass,
|
||||||
disk::{
|
disk::{
|
||||||
@@ -174,14 +174,15 @@ use std::future::Future;
|
|||||||
use std::hash::{BuildHasher, Hash, Hasher};
|
use std::hash::{BuildHasher, Hash, Hasher};
|
||||||
use std::mem::{self};
|
use std::mem::{self};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
|
use std::sync::OnceLock;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, OnceLock};
|
|
||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll};
|
||||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet},
|
collections::{HashMap, HashSet},
|
||||||
io::{Cursor, Write},
|
io::{Cursor, Write},
|
||||||
path::Path,
|
path::Path,
|
||||||
|
sync::Arc,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
@@ -620,9 +621,7 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
|
|||||||
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
|
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
||||||
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
||||||
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
||||||
@@ -690,36 +689,22 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102
|
|||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
|
||||||
// Enabled by default (backlog#872): the early-stop path only engages for
|
// Enabled by default (backlog#872): the early-stop path only engages for
|
||||||
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version
|
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version
|
||||||
// reads by default, without version_id / healing / free-version needs) and still
|
// metadata-only reads by default, without version_id / healing / free-version
|
||||||
// requires a full read-quorum agreement before stopping. Data-read requests add
|
// needs) and still requires a full read-quorum agreement before stopping. Set
|
||||||
// a separate inline-shard verifier before cancelling the remaining fanout. Set
|
|
||||||
// the env var to `false` to fall back to full-wait metadata fanout.
|
// the env var to `false` to fall back to full-wait metadata fanout.
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
|
||||||
)]
|
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true;
|
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = false;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";
|
|
||||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS";
|
|
||||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET";
|
|
||||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX";
|
|
||||||
|
|
||||||
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
||||||
|
|
||||||
@@ -921,16 +906,18 @@ mod prepared_get_object_metadata_tests {
|
|||||||
.expect("test should find an object whose initial fanout covers both data shards")
|
.expect("test should find an object whose initial fanout covers both data shards")
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||||
.get(3)
|
.get(3)
|
||||||
.expect("4-disk test geometry should leave one bounded spare disk")
|
.expect("4-disk test geometry should leave one bounded spare disk")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize {
|
||||||
|
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||||
|
.get(2)
|
||||||
|
.expect("4-disk test geometry should include a third initial metadata disk")
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||||
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
|
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
|
||||||
@@ -1049,7 +1036,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial(body_cache_hook)]
|
#[serial_test::serial(body_cache_hook)]
|
||||||
fn inline_data_read_early_stop_defaults_return_exact_body() {
|
fn inline_data_read_early_stop_reader_returns_exact_body() {
|
||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()
|
.build()
|
||||||
@@ -1081,14 +1068,14 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
temp_env::async_with_vars(
|
temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
|
let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object);
|
||||||
let barrier =
|
let barrier =
|
||||||
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
let calls = disk_call_counters::observe(&object);
|
let calls = disk_call_counters::observe(&object);
|
||||||
let set_disks_for_read = Arc::clone(&set_disks);
|
let set_disks_for_read = Arc::clone(&set_disks);
|
||||||
let opts_for_read = opts.clone();
|
let opts_for_read = opts.clone();
|
||||||
@@ -1101,10 +1088,10 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
|
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
|
||||||
.await
|
.await
|
||||||
.expect("default inline GET should pause a slow parity metadata read");
|
.expect("bounded inline GET should pause a slow initial metadata read");
|
||||||
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
|
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
|
||||||
.await
|
.await
|
||||||
.expect("default production inline GET should return before the paused parity metadata response")
|
.expect("production inline GET should return before the paused metadata response")
|
||||||
.expect("inline GET reader task should not panic")
|
.expect("inline GET reader task should not panic")
|
||||||
.expect("inline GET reader should open");
|
.expect("inline GET reader should open");
|
||||||
let object_size = reader.object_info.size;
|
let object_size = reader.object_info.size;
|
||||||
@@ -1125,17 +1112,14 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
assert_eq!(object_size, payload.len() as i64);
|
assert_eq!(object_size, payload.len() as i64);
|
||||||
assert_eq!(restored, payload);
|
assert_eq!(restored, payload);
|
||||||
assert_eq!(
|
assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare");
|
||||||
calls_total, 4,
|
|
||||||
"default production inline GET should schedule the initial bounded quorum plus one hedge"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![4.0],
|
vec![4.0],
|
||||||
"default production GET should record all scheduled metadata tasks"
|
"bounded production GET should record all scheduled metadata tasks"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -1143,7 +1127,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![3.0],
|
vec![3.0],
|
||||||
"default production GET should record only observed metadata responses as completed"
|
"bounded production GET should record only observed metadata responses as completed"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -1151,7 +1135,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![1.0],
|
vec![1.0],
|
||||||
"default production GET should record the aborted slow parity metadata task"
|
"bounded production GET should record the aborted slow metadata task"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1298,9 +1282,9 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
temp_env::async_with_vars(
|
temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
let calls = disk_call_counters::observe(&object);
|
let calls = disk_call_counters::observe(&object);
|
||||||
@@ -1702,95 +1686,6 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct GetMetadataSlowtailFaultConfig {
|
|
||||||
delay: Duration,
|
|
||||||
disks: Arc<[usize]>,
|
|
||||||
bucket: Option<String>,
|
|
||||||
object_prefix: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct GetMetadataSlowtailFaultRequest {
|
|
||||||
delay: Duration,
|
|
||||||
disks: Arc<[usize]>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl GetMetadataSlowtailFaultRequest {
|
|
||||||
fn delay_for_disk(&self, disk_index: usize) -> Option<Duration> {
|
|
||||||
self.disks.contains(&disk_index).then_some(self.delay)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option<Vec<usize>> {
|
|
||||||
let mut disks = Vec::new();
|
|
||||||
for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) {
|
|
||||||
let Ok(index) = item.parse::<usize>() else {
|
|
||||||
return None;
|
|
||||||
};
|
|
||||||
if !disks.contains(&index) {
|
|
||||||
disks.push(index);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
(!disks.is_empty()).then_some(disks)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn load_get_metadata_slowtail_fault_config() -> Option<GetMetadataSlowtailFaultConfig> {
|
|
||||||
let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0);
|
|
||||||
if delay_ms == 0 {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?;
|
|
||||||
let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET)
|
|
||||||
.ok()
|
|
||||||
.filter(|value| !value.is_empty());
|
|
||||||
let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX)
|
|
||||||
.ok()
|
|
||||||
.filter(|value| !value.is_empty());
|
|
||||||
Some(GetMetadataSlowtailFaultConfig {
|
|
||||||
delay: Duration::from_millis(delay_ms),
|
|
||||||
disks: Arc::from(disks.into_boxed_slice()),
|
|
||||||
bucket,
|
|
||||||
object_prefix,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option<GetMetadataSlowtailFaultRequest> {
|
|
||||||
if !read_data {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
let config = load_get_metadata_slowtail_fault_config();
|
|
||||||
#[cfg(test)]
|
|
||||||
let config = config.as_ref()?;
|
|
||||||
#[cfg(not(test))]
|
|
||||||
let config = ({
|
|
||||||
static CACHED: OnceLock<Option<GetMetadataSlowtailFaultConfig>> = OnceLock::new();
|
|
||||||
CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref()
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if let Some(expected_bucket) = &config.bucket
|
|
||||||
&& expected_bucket != bucket
|
|
||||||
{
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
if let Some(expected_prefix) = &config.object_prefix
|
|
||||||
&& !object.starts_with(expected_prefix)
|
|
||||||
{
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(GetMetadataSlowtailFaultRequest {
|
|
||||||
delay: config.delay,
|
|
||||||
disks: config.disks.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option<Duration> {
|
|
||||||
get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
||||||
/// while the current part decodes (backlog#870).
|
/// while the current part decodes (backlog#870).
|
||||||
///
|
///
|
||||||
@@ -1816,10 +1711,6 @@ fn is_multipart_reader_setup_prefetch_enabled() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn get_metadata_early_stop_rollout_pct() -> u32 {
|
fn get_metadata_early_stop_rollout_pct() -> u32 {
|
||||||
static CACHED: OnceLock<u32> = OnceLock::new();
|
static CACHED: OnceLock<u32> = OnceLock::new();
|
||||||
*CACHED.get_or_init(|| {
|
*CACHED.get_or_init(|| {
|
||||||
@@ -1859,10 +1750,6 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Should this specific request use metadata early-stop?
|
/// Should this specific request use metadata early-stop?
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
|
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
|
||||||
let base = is_get_metadata_early_stop_enabled();
|
let base = is_get_metadata_early_stop_enabled();
|
||||||
let pct = get_metadata_early_stop_rollout_pct();
|
let pct = get_metadata_early_stop_rollout_pct();
|
||||||
@@ -2296,7 +2183,6 @@ fn classify_get_codec_streaming_object_class(
|
|||||||
GetCodecStreamingObjectClass::PlainSinglePart
|
GetCodecStreamingObjectClass::PlainSinglePart
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn is_get_small_object_direct_memory_eligible_with_threshold(
|
fn is_get_small_object_direct_memory_eligible_with_threshold(
|
||||||
range: &Option<HTTPRangeSpec>,
|
range: &Option<HTTPRangeSpec>,
|
||||||
object_info: &ObjectInfo,
|
object_info: &ObjectInfo,
|
||||||
@@ -2902,7 +2788,6 @@ pub struct SetDisks {
|
|||||||
/// Stable namespace shared by every object lock created for this set.
|
/// Stable namespace shared by every object lock created for this set.
|
||||||
set_lock_namespace: Arc<str>,
|
set_lock_namespace: Arc<str>,
|
||||||
pub format: FormatV3,
|
pub format: FormatV3,
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||||
@@ -3178,13 +3063,11 @@ struct GetObjectMetadataCacheEntry {
|
|||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct DiskHealthEntry {
|
struct DiskHealthEntry {
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
last_check: Instant,
|
last_check: Instant,
|
||||||
online: bool,
|
online: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DiskHealthEntry {
|
impl DiskHealthEntry {
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn cached_value(&self) -> Option<bool> {
|
fn cached_value(&self) -> Option<bool> {
|
||||||
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
|
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
|
||||||
Some(self.online)
|
Some(self.online)
|
||||||
@@ -3778,7 +3661,6 @@ fn multipart_put_large_batch_min_size_bytes() -> usize {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
|
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
|
||||||
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
|
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
|
||||||
SmallWritePath::Inline
|
SmallWritePath::Inline
|
||||||
@@ -3981,20 +3863,11 @@ fn inline_erasure_shard_file_offset(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn collect_inline_data_shard_fileinfos_by_index<'a>(
|
fn collect_inline_data_shard_fileinfos_by_index<'a>(
|
||||||
parts_metadata: &'a [FileInfo],
|
|
||||||
fi: &FileInfo,
|
|
||||||
data_shards: usize,
|
|
||||||
disk_is_online: impl FnMut(usize) -> bool,
|
|
||||||
) -> Option<Vec<&'a FileInfo>> {
|
|
||||||
collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, fi, data_shards, disk_is_online).ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
|
|
||||||
parts_metadata: &'a [FileInfo],
|
parts_metadata: &'a [FileInfo],
|
||||||
fi: &FileInfo,
|
fi: &FileInfo,
|
||||||
data_shards: usize,
|
data_shards: usize,
|
||||||
mut disk_is_online: impl FnMut(usize) -> bool,
|
mut disk_is_online: impl FnMut(usize) -> bool,
|
||||||
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> {
|
) -> Option<Vec<&'a FileInfo>> {
|
||||||
let distribution = &fi.erasure.distribution;
|
let distribution = &fi.erasure.distribution;
|
||||||
let mut data_files = vec![None; data_shards];
|
let mut data_files = vec![None; data_shards];
|
||||||
|
|
||||||
@@ -4002,35 +3875,27 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
|
|||||||
if !disk_is_online(disk_index) {
|
if !disk_is_online(disk_index) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let Some(&block_index) = distribution.get(disk_index) else {
|
let block_index = *distribution.get(disk_index)?;
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
|
||||||
};
|
|
||||||
if block_index == 0 || block_index > data_shards {
|
if block_index == 0 || block_index > data_shards {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if file_info.name.is_empty() {
|
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD);
|
|
||||||
}
|
|
||||||
if file_info.erasure.index != block_index {
|
if file_info.erasure.index != block_index {
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH);
|
continue;
|
||||||
}
|
}
|
||||||
if !file_info.has_valid_erasure_geometry() {
|
if !file_info.has_valid_erasure_geometry() {
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
continue;
|
||||||
}
|
}
|
||||||
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
|
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH);
|
continue;
|
||||||
}
|
}
|
||||||
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
|
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
|
||||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD);
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
data_files[block_index - 1] = Some(file_info);
|
data_files[block_index - 1] = Some(file_info);
|
||||||
}
|
}
|
||||||
|
|
||||||
data_files
|
data_files.into_iter().collect()
|
||||||
.into_iter()
|
|
||||||
.collect::<Option<Vec<_>>>()
|
|
||||||
.ok_or(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
@@ -4357,7 +4222,6 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
|
|||||||
///
|
///
|
||||||
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so
|
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so
|
||||||
/// object-lock protection is never skipped because of a metadata lookup miss.
|
/// object-lock protection is never skipped because of a metadata lookup miss.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
|
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
|
||||||
bucket_meta.is_none_or(|meta| meta.object_locking())
|
bucket_meta.is_none_or(|meta| meta.object_locking())
|
||||||
}
|
}
|
||||||
@@ -4633,6 +4497,15 @@ impl Hash for ObjProps {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone, Debug)]
|
||||||
|
pub struct HealEntryResult {
|
||||||
|
pub bytes: usize,
|
||||||
|
pub success: bool,
|
||||||
|
pub skipped: bool,
|
||||||
|
pub entry_done: bool,
|
||||||
|
pub name: String,
|
||||||
|
}
|
||||||
|
|
||||||
fn is_object_dangling(
|
fn is_object_dangling(
|
||||||
meta_arr: &[FileInfo],
|
meta_arr: &[FileInfo],
|
||||||
errs: &[Option<DiskError>],
|
errs: &[Option<DiskError>],
|
||||||
@@ -5409,7 +5282,6 @@ pub fn is_valid_storage_class(storage_class: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns true if the storage class is a cold storage tier that requires special handling
|
/// Returns true if the storage class is a cold storage tier that requires special handling
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
storage_class,
|
storage_class,
|
||||||
@@ -5418,7 +5290,6 @@ pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns true if the storage class is an infrequent access tier
|
/// Returns true if the storage class is an infrequent access tier
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
storage_class,
|
storage_class,
|
||||||
@@ -6998,100 +6869,6 @@ mod tests {
|
|||||||
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn recv_abandoned_parts_trace(
|
|
||||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
state: &str,
|
|
||||||
) -> rustfs_common::trace_bus::TraceEvent {
|
|
||||||
for _ in 0..32 {
|
|
||||||
let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv())
|
|
||||||
.await
|
|
||||||
.expect("abandoned-parts trace event should arrive")
|
|
||||||
.expect("trace bus should stay open");
|
|
||||||
if event.kind == rustfs_common::trace_bus::TraceKind::Heal
|
|
||||||
&& event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts
|
|
||||||
&& event.bucket.as_deref() == Some(bucket)
|
|
||||||
&& event.object.as_deref() == Some(object)
|
|
||||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
|
||||||
{
|
|
||||||
return (*event).clone();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
panic!("expected abandoned-parts trace state {state} for {bucket}/{object}");
|
|
||||||
}
|
|
||||||
|
|
||||||
fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option<String> {
|
|
||||||
event.attrs.iter().find_map(|attr| {
|
|
||||||
if attr.key != key {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(match &attr.value {
|
|
||||||
rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(),
|
|
||||||
rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(),
|
|
||||||
rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(),
|
|
||||||
rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn check_abandoned_parts_dry_run_counts_without_deleting() {
|
|
||||||
let mut trace = rustfs_common::trace_bus::subscribe_trace_events();
|
|
||||||
let (dir, disk) = make_single_local_disk().await;
|
|
||||||
let live = Uuid::new_v4();
|
|
||||||
let orphan = Uuid::new_v4();
|
|
||||||
|
|
||||||
let object_dir = dir.path().join("bucket").join("obj");
|
|
||||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
|
||||||
fs::create_dir_all(object_dir.join(live.to_string()))
|
|
||||||
.await
|
|
||||||
.expect("live data dir should be created");
|
|
||||||
fs::create_dir_all(object_dir.join(orphan.to_string()))
|
|
||||||
.await
|
|
||||||
.expect("orphan data dir should be created");
|
|
||||||
|
|
||||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
|
||||||
set.check_abandoned_parts(
|
|
||||||
"bucket",
|
|
||||||
"obj",
|
|
||||||
&HealOpts {
|
|
||||||
dry_run: true,
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("dry-run abandoned-parts check should succeed");
|
|
||||||
let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await;
|
|
||||||
assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true"));
|
|
||||||
assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1"));
|
|
||||||
|
|
||||||
assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved");
|
|
||||||
assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir");
|
|
||||||
|
|
||||||
set.check_abandoned_parts(
|
|
||||||
"bucket",
|
|
||||||
"obj",
|
|
||||||
&HealOpts {
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("abandoned-parts check should reclaim stale data dir");
|
|
||||||
let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await;
|
|
||||||
assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false"));
|
|
||||||
assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1"));
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
object_dir.join(live.to_string()).exists(),
|
|
||||||
"referenced data dir must remain after reclaim"
|
|
||||||
);
|
|
||||||
assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
|
async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
|
||||||
let (dir, disk) = make_single_local_disk().await;
|
let (dir, disk) = make_single_local_disk().await;
|
||||||
@@ -12327,18 +12104,11 @@ mod tests {
|
|||||||
.expect_err("unsupported copy_object_part should return a typed error");
|
.expect_err("unsupported copy_object_part should return a typed error");
|
||||||
assert!(matches!(copy_part_err, StorageError::NotImplemented));
|
assert!(matches!(copy_part_err, StorageError::NotImplemented));
|
||||||
|
|
||||||
set_disks
|
let abandoned_err = set_disks
|
||||||
.check_abandoned_parts(
|
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||||
"bucket",
|
|
||||||
"object",
|
|
||||||
&HealOpts {
|
|
||||||
dry_run: true,
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.expect("abandoned-parts check should be callable on empty disk sets");
|
.expect_err("abandoned-parts check should stay in the upper reconciliation layer");
|
||||||
|
assert!(matches!(abandoned_err, StorageError::NotImplemented));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ use super::super::*;
|
|||||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
|
||||||
use tracing::trace;
|
use tracing::trace;
|
||||||
|
|
||||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||||
@@ -454,9 +453,7 @@ impl SetDisks {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Bound, not `_`: this guard must live to the end of the scope. A bare
|
let write_lock_guard = if !opts.no_lock {
|
||||||
// `_` would drop it here and release the namespace write lock.
|
|
||||||
let _write_lock_guard = if !opts.no_lock {
|
|
||||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||||
Some(
|
Some(
|
||||||
ns_lock
|
ns_lock
|
||||||
@@ -999,7 +996,7 @@ impl SetDisks {
|
|||||||
readers.push(None);
|
readers.push(None);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Err(_e) => {
|
Err(e) => {
|
||||||
readers.push(None);
|
readers.push(None);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1548,9 +1545,6 @@ impl SetDisks {
|
|||||||
|
|
||||||
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
||||||
for (disk_index, disk) in disks.iter().enumerate() {
|
for (disk_index, disk) in disks.iter().enumerate() {
|
||||||
// Only the #[cfg(test)] fault-injection branch below reads this.
|
|
||||||
#[cfg(not(test))]
|
|
||||||
let _ = disk_index;
|
|
||||||
let Some(disk) = disk else {
|
let Some(disk) = disk else {
|
||||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||||
};
|
};
|
||||||
@@ -1722,10 +1716,6 @@ impl SetDisks {
|
|||||||
Ok((result, None))
|
Ok((result, None))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "lock-taking wrapper over the live heal_object_dir_locked; only comments reference it (backlog#1823)"
|
|
||||||
)]
|
|
||||||
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
||||||
pub(in crate::set_disk) async fn heal_object_dir(
|
pub(in crate::set_disk) async fn heal_object_dir(
|
||||||
&self,
|
&self,
|
||||||
@@ -2058,61 +2048,11 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
|||||||
Err(Error::DiskNotFound)
|
Err(Error::DiskNotFound)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||||
let started_at = std::time::Instant::now();
|
// Multipart orphan reconciliation is intentionally retained above the set layer
|
||||||
let _write_lock_guard = if !opts.no_lock {
|
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
Err(StorageError::NotImplemented)
|
||||||
Some(
|
|
||||||
ns_lock
|
|
||||||
.get_write_lock(get_lock_acquire_timeout())
|
|
||||||
.await
|
|
||||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let removed = if opts.dry_run {
|
|
||||||
self.dry_run_reclaim_orphan_data_dirs(bucket, object).await?
|
|
||||||
} else {
|
|
||||||
self.reclaim_orphan_data_dirs(bucket, object).await?
|
|
||||||
};
|
|
||||||
let state = if opts.dry_run && removed > 0 {
|
|
||||||
"dry_run_matched"
|
|
||||||
} else if removed > 0 {
|
|
||||||
"reclaimed"
|
|
||||||
} else {
|
|
||||||
"checked"
|
|
||||||
};
|
|
||||||
let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX);
|
|
||||||
|
|
||||||
trace_emit(|| {
|
|
||||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts)
|
|
||||||
.with_bucket(bucket)
|
|
||||||
.with_object(object)
|
|
||||||
.with_duration(started_at.elapsed())
|
|
||||||
.with_attr("state", state)
|
|
||||||
.with_attr("dry_run", opts.dry_run)
|
|
||||||
.with_attr("data_dirs", data_dirs)
|
|
||||||
});
|
|
||||||
|
|
||||||
if removed > 0 {
|
|
||||||
trace!(
|
|
||||||
event = "heal_abandoned_parts",
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
|
||||||
state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" },
|
|
||||||
result = "ok",
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
dry_run = opts.dry_run,
|
|
||||||
data_dirs = removed,
|
|
||||||
"Heal abandoned parts checked object data directories"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3297,223 +3237,4 @@ mod heal_result_report_tests {
|
|||||||
assert!(result.detail.contains("part 1"));
|
assert!(result.detail.contains("part 1"));
|
||||||
assert!(result.detail.contains("bitrot_failure=true"));
|
assert!(result.detail.contains("bitrot_failure=true"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// HS-12 (backlog#1874): a versioned DELETE racing an object heal must never
|
|
||||||
// resurrect the deleted version. The heal has real reconstruction work (a
|
|
||||||
// shard of the doomed version is removed), so both sides touch the same
|
|
||||||
// (bucket, object, data_dir); whichever order the ns write lock serializes
|
|
||||||
// them in, the committed delete must win.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn heal_racing_version_delete_never_resurrects_the_deleted_version() {
|
|
||||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
|
||||||
let bucket = "heal-race-delete-no-resurrect";
|
|
||||||
let object = "object.bin";
|
|
||||||
set.make_bucket(
|
|
||||||
bucket,
|
|
||||||
&MakeBucketOptions {
|
|
||||||
versioning_enabled: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("versioned bucket should be created");
|
|
||||||
|
|
||||||
let mut first_reader = PutObjReader::from_vec(vec![0x11; 1024 * 1024]);
|
|
||||||
let first_info = set
|
|
||||||
.put_object(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&mut first_reader,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("first version should be written");
|
|
||||||
let first_version = first_info
|
|
||||||
.version_id
|
|
||||||
.expect("versioned put should return the first version id")
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let mut second_reader = PutObjReader::from_vec(vec![0x22; 1024 * 1024]);
|
|
||||||
let second_info = set
|
|
||||||
.put_object(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&mut second_reader,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("second version should be written");
|
|
||||||
let second_version = second_info
|
|
||||||
.version_id
|
|
||||||
.expect("versioned put should return the second version id")
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
// Damage one shard of the doomed version so the racing heal performs an
|
|
||||||
// actual reconstruction over its data dir instead of an early exit.
|
|
||||||
let doomed_source = disks[0]
|
|
||||||
.read_version("", bucket, object, &first_version, &ReadOptions::default())
|
|
||||||
.await
|
|
||||||
.expect("doomed version metadata should be readable");
|
|
||||||
let doomed_data_dir = doomed_source
|
|
||||||
.data_dir
|
|
||||||
.expect("non-inline version should have a data directory");
|
|
||||||
tokio::fs::remove_file(
|
|
||||||
temp_dirs[1]
|
|
||||||
.path()
|
|
||||||
.join(bucket)
|
|
||||||
.join(object)
|
|
||||||
.join(doomed_data_dir.to_string())
|
|
||||||
.join("part.1"),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("shard damage should be injected before the race");
|
|
||||||
|
|
||||||
let delete_set = set.clone();
|
|
||||||
let (delete_res, heal_res) = tokio::join!(
|
|
||||||
async {
|
|
||||||
delete_set
|
|
||||||
.delete_object(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
ObjectOptions {
|
|
||||||
versioned: true,
|
|
||||||
version_id: Some(first_version.clone()),
|
|
||||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
|
||||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
|
||||||
))),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
},
|
|
||||||
async {
|
|
||||||
set.heal_object(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
"",
|
|
||||||
&HealOpts {
|
|
||||||
scan_mode: HealScanMode::Deep,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
},
|
|
||||||
);
|
|
||||||
delete_res.expect("version delete must succeed under lock serialization");
|
|
||||||
// The heal may legitimately report a transient failure when the version
|
|
||||||
// it was rebuilding disappears mid-flight; only the end state matters.
|
|
||||||
drop(heal_res);
|
|
||||||
|
|
||||||
let resurrected = set
|
|
||||||
.get_object_info(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned: true,
|
|
||||||
version_id: Some(first_version.clone()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert!(
|
|
||||||
matches!(&resurrected, Err(Error::FileVersionNotFound) | Err(Error::ObjectNotFound(..))),
|
|
||||||
"a racing heal must not resurrect the deleted version: {resurrected:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
let survivor = set
|
|
||||||
.get_object_info(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned: true,
|
|
||||||
version_id: Some(second_version.clone()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("surviving version must remain readable after the race");
|
|
||||||
assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact");
|
|
||||||
}
|
|
||||||
|
|
||||||
// HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on
|
|
||||||
// the same object. The overwrite's post-commit tail deletes the replaced
|
|
||||||
// data dir without the ns lock (object.rs commit tail), which is exactly
|
|
||||||
// the intersection the audit flagged: the heal must tolerate the tail race
|
|
||||||
// (retryable outcome) and every committed overwrite must survive — the
|
|
||||||
// final current version is exactly the last payload written.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() {
|
|
||||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
|
||||||
let bucket = "heal-race-put-overwrite";
|
|
||||||
let object = "object.bin";
|
|
||||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
|
||||||
.await
|
|
||||||
.expect("bucket should be created");
|
|
||||||
|
|
||||||
const ROUNDS: usize = 8;
|
|
||||||
const PAYLOAD_SIZE: usize = 256 * 1024;
|
|
||||||
let mut last_etag = String::new();
|
|
||||||
for round in 0..ROUNDS {
|
|
||||||
// Give the heal something to rebuild on alternating rounds: remove a
|
|
||||||
// shard of the current data dir right before the race.
|
|
||||||
if round % 2 == 1 {
|
|
||||||
let current = disks[2]
|
|
||||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
|
||||||
.await
|
|
||||||
.expect("current metadata should be readable");
|
|
||||||
if let Some(data_dir) = current.data_dir {
|
|
||||||
let shard = temp_dirs[3]
|
|
||||||
.path()
|
|
||||||
.join(bucket)
|
|
||||||
.join(object)
|
|
||||||
.join(data_dir.to_string())
|
|
||||||
.join("part.1");
|
|
||||||
if shard.exists() {
|
|
||||||
tokio::fs::remove_file(&shard)
|
|
||||||
.await
|
|
||||||
.expect("shard damage should be injectable mid-race");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let payload = vec![round as u8; PAYLOAD_SIZE];
|
|
||||||
let mut put_reader = PutObjReader::from_vec(payload);
|
|
||||||
let put_opts = ObjectOptions::default();
|
|
||||||
let heal_opts = HealOpts {
|
|
||||||
scan_mode: HealScanMode::Deep,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let (put_res, heal_res) = tokio::join!(
|
|
||||||
set.put_object(bucket, object, &mut put_reader, &put_opts),
|
|
||||||
set.heal_object(bucket, object, "", &heal_opts),
|
|
||||||
);
|
|
||||||
let put_info = put_res.expect("overwrite must succeed under lock serialization");
|
|
||||||
last_etag = put_info.etag.clone().unwrap_or_default();
|
|
||||||
// Heal outcome is unconstrained (may hit the tail race and report a
|
|
||||||
// retryable error); the invariant is checked on the end state.
|
|
||||||
drop(heal_res);
|
|
||||||
}
|
|
||||||
|
|
||||||
let final_info = set
|
|
||||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
|
||||||
.await
|
|
||||||
.expect("object must remain readable after the race loop");
|
|
||||||
assert_eq!(
|
|
||||||
final_info.size, PAYLOAD_SIZE as i64,
|
|
||||||
"final current version must be the last committed overwrite"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
final_info.etag.unwrap_or_default(),
|
|
||||||
last_etag,
|
|
||||||
"the racing heal loop must never leave a stale or resurrected current version"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,6 @@
|
|||||||
//! per-version `SetDisks::heal_object`.
|
//! per-version `SetDisks::heal_object`.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::*;
|
||||||
use crate::object_api::ObjectInfo;
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
@@ -40,16 +39,12 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
|||||||
/// it must not gate healing logic — the delete-marker vs data path is chosen
|
/// it must not gate healing logic — the delete-marker vs data path is chosen
|
||||||
/// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is
|
/// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is
|
||||||
/// normalized (nil/absent UUID => `None`).
|
/// normalized (nil/absent UUID => `None`).
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct HealWalkVersion {
|
pub struct HealWalkVersion {
|
||||||
/// object key
|
/// object key
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// normalized version id (`None` when the version is nil/absent)
|
/// normalized version id (`None` when the version is nil/absent)
|
||||||
pub version_id: Option<String>,
|
pub version_id: Option<String>,
|
||||||
/// version modification time as Unix nanoseconds
|
|
||||||
pub mod_time_unix_nanos: Option<i128>,
|
|
||||||
/// object snapshot for lifecycle evaluation
|
|
||||||
pub lifecycle_object_info: Option<ObjectInfo>,
|
|
||||||
/// whether this version is a delete marker (observability only)
|
/// whether this version is a delete marker (observability only)
|
||||||
pub is_delete_marker: bool,
|
pub is_delete_marker: bool,
|
||||||
}
|
}
|
||||||
@@ -68,7 +63,6 @@ struct HealWalkCollector {
|
|||||||
bucket: String,
|
bucket: String,
|
||||||
batch_objects: usize,
|
batch_objects: usize,
|
||||||
version_budget: usize,
|
version_budget: usize,
|
||||||
include_lifecycle_object_info: bool,
|
|
||||||
objects: Mutex<Vec<HealWalkObject>>,
|
objects: Mutex<Vec<HealWalkObject>>,
|
||||||
decode_error: Mutex<Option<DiskError>>,
|
decode_error: Mutex<Option<DiskError>>,
|
||||||
version_total: AtomicUsize,
|
version_total: AtomicUsize,
|
||||||
@@ -122,25 +116,10 @@ impl HealWalkCollector {
|
|||||||
|
|
||||||
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
|
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
|
||||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
|
||||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
|
||||||
let mut lifecycle_fi = fi.clone();
|
|
||||||
lifecycle_fi.version_id = version_uuid;
|
|
||||||
Some(ObjectInfo::from_file_info(
|
|
||||||
&lifecycle_fi,
|
|
||||||
&self.bucket,
|
|
||||||
&entry.name,
|
|
||||||
version_uuid.is_some(),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
versions.push(HealWalkVersion {
|
versions.push(HealWalkVersion {
|
||||||
name: entry.name.clone(),
|
name: entry.name.clone(),
|
||||||
// Normalize: nil/absent version id => None.
|
// Normalize: nil/absent version id => None.
|
||||||
version_id: version_uuid.map(|u| u.to_string()),
|
version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
|
||||||
lifecycle_object_info,
|
|
||||||
is_delete_marker: fi.deleted,
|
is_delete_marker: fi.deleted,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -194,26 +173,11 @@ impl HealWalkCollector {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string());
|
||||||
let vid = version_uuid.map(|u| u.to_string());
|
|
||||||
if seen.insert(vid.clone()) {
|
if seen.insert(vid.clone()) {
|
||||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
|
||||||
let mut lifecycle_fi = fi.clone();
|
|
||||||
lifecycle_fi.version_id = version_uuid;
|
|
||||||
Some(ObjectInfo::from_file_info(
|
|
||||||
&lifecycle_fi,
|
|
||||||
&self.bucket,
|
|
||||||
&entry.name,
|
|
||||||
version_uuid.is_some(),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
versions.push(HealWalkVersion {
|
versions.push(HealWalkVersion {
|
||||||
name: entry.name.clone(),
|
name: entry.name.clone(),
|
||||||
version_id: vid,
|
version_id: vid,
|
||||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
|
||||||
lifecycle_object_info,
|
|
||||||
is_delete_marker: fi.deleted,
|
is_delete_marker: fi.deleted,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -291,7 +255,6 @@ impl SetDisks {
|
|||||||
forward_to: Option<&str>,
|
forward_to: Option<&str>,
|
||||||
batch_objects: usize,
|
batch_objects: usize,
|
||||||
version_budget: usize,
|
version_budget: usize,
|
||||||
include_lifecycle_object_info: bool,
|
|
||||||
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||||
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
|
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
|
||||||
|
|
||||||
@@ -301,7 +264,6 @@ impl SetDisks {
|
|||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
batch_objects,
|
batch_objects,
|
||||||
version_budget: version_budget.max(1),
|
version_budget: version_budget.max(1),
|
||||||
include_lifecycle_object_info,
|
|
||||||
objects: Mutex::new(Vec::new()),
|
objects: Mutex::new(Vec::new()),
|
||||||
decode_error: Mutex::new(None),
|
decode_error: Mutex::new(None),
|
||||||
version_total: AtomicUsize::new(0),
|
version_total: AtomicUsize::new(0),
|
||||||
@@ -385,7 +347,6 @@ mod tests {
|
|||||||
bucket: "bucket".to_string(),
|
bucket: "bucket".to_string(),
|
||||||
batch_objects: 2,
|
batch_objects: 2,
|
||||||
version_budget: 2,
|
version_budget: 2,
|
||||||
include_lifecycle_object_info: false,
|
|
||||||
objects: Mutex::new(Vec::new()),
|
objects: Mutex::new(Vec::new()),
|
||||||
decode_error: Mutex::new(None),
|
decode_error: Mutex::new(None),
|
||||||
version_total: AtomicUsize::new(0),
|
version_total: AtomicUsize::new(0),
|
||||||
@@ -427,8 +388,6 @@ mod tests {
|
|||||||
HealWalkVersion {
|
HealWalkVersion {
|
||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
version_id: Some(id.to_string()),
|
version_id: Some(id.to_string()),
|
||||||
mod_time_unix_nanos: None,
|
|
||||||
lifecycle_object_info: None,
|
|
||||||
is_delete_marker: dm,
|
is_delete_marker: dm,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -532,7 +491,6 @@ mod tests {
|
|||||||
bucket: "bucket".to_string(),
|
bucket: "bucket".to_string(),
|
||||||
batch_objects: 1000,
|
batch_objects: 1000,
|
||||||
version_budget: 10_000,
|
version_budget: 10_000,
|
||||||
include_lifecycle_object_info: false,
|
|
||||||
objects: Mutex::new(Vec::new()),
|
objects: Mutex::new(Vec::new()),
|
||||||
version_total: AtomicUsize::new(0),
|
version_total: AtomicUsize::new(0),
|
||||||
decode_error: Mutex::new(None),
|
decode_error: Mutex::new(None),
|
||||||
@@ -609,7 +567,7 @@ mod tests {
|
|||||||
.expect("corrupt test metadata should be written");
|
.expect("corrupt test metadata should be written");
|
||||||
|
|
||||||
let error = set_disks
|
let error = set_disks
|
||||||
.heal_walk_versions_page(bucket, "", None, 2, 2, false)
|
.heal_walk_versions_page(bucket, "", None, 2, 2)
|
||||||
.await
|
.await
|
||||||
.expect_err("semantic metadata corruption must fail the heal disk walk");
|
.expect_err("semantic metadata corruption must fail the heal disk walk");
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,6 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||||
match err {
|
match err {
|
||||||
LockResult::Timeout => {
|
LockResult::Timeout => {
|
||||||
@@ -80,7 +79,6 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) fn format_lock_error_from_error(
|
pub(in crate::set_disk) fn format_lock_error_from_error(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -145,7 +143,6 @@ impl SetDisks {
|
|||||||
disks
|
disks
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
||||||
let snapshot = self.drive_membership_snapshot().await;
|
let snapshot = self.drive_membership_snapshot().await;
|
||||||
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
|
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
|
||||||
@@ -156,10 +153,6 @@ impl SetDisks {
|
|||||||
disks
|
disks
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "local-only sibling of the test-covered get_online_disks; no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||||
let snapshot = self.drive_membership_snapshot().await;
|
let snapshot = self.drive_membership_snapshot().await;
|
||||||
let mut disks = snapshot
|
let mut disks = snapshot
|
||||||
@@ -439,10 +432,6 @@ impl SetDisks {
|
|||||||
Ok((disk, fm))
|
Ok((disk, fm))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "MinIO-parity healing-disk accessor with no caller in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(in crate::set_disk) async fn get_online_disk_with_healing(
|
pub(in crate::set_disk) async fn get_online_disk_with_healing(
|
||||||
&self,
|
&self,
|
||||||
incl_healing: bool,
|
incl_healing: bool,
|
||||||
@@ -451,10 +440,6 @@ impl SetDisks {
|
|||||||
Ok((new_disks, healing > 0))
|
Ok((new_disks, healing > 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "reached only from get_online_disk_with_healing, itself uncalled in this port (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
|
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
|
||||||
&self,
|
&self,
|
||||||
incl_healing: bool,
|
incl_healing: bool,
|
||||||
|
|||||||
@@ -415,7 +415,6 @@ fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize
|
|||||||
/// never returned, but flips `is_truncated` to `true` and yields a
|
/// never returned, but flips `is_truncated` to `true` and yields a
|
||||||
/// `next_upload_id_marker` pointing at the last returned upload so the caller can
|
/// `next_upload_id_marker` pointing at the last returned upload so the caller can
|
||||||
/// resume paging.
|
/// resume paging.
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
|
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
|
||||||
let is_truncated = remaining.len() > max_uploads;
|
let is_truncated = remaining.len() > max_uploads;
|
||||||
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
|
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
|
||||||
@@ -558,7 +557,6 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(self))]
|
#[tracing::instrument(level = "debug", skip(self))]
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
pub(super) async fn check_upload_id_exists(
|
pub(super) async fn check_upload_id_exists(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -1400,7 +1398,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
let mut count = max_parts;
|
let mut count = max_parts;
|
||||||
|
|
||||||
for part in object_parts.iter() {
|
for (i, part) in object_parts.iter().enumerate() {
|
||||||
if let Some(err) = &part.error {
|
if let Some(err) = &part.error {
|
||||||
warn!("list_object_parts part error: {:?}", &err);
|
warn!("list_object_parts part error: {:?}", &err);
|
||||||
}
|
}
|
||||||
@@ -2043,8 +2041,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
||||||
{
|
{
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}, err={}",
|
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
|
||||||
p.part_num, bucket, object, err
|
p.part_num, bucket, object
|
||||||
);
|
);
|
||||||
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
||||||
}
|
}
|
||||||
@@ -2089,8 +2087,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
|
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload checksum matches failed want={}, got={}, err={}",
|
"complete_multipart_upload checksum matches failed want={}, got={}",
|
||||||
wtcs.encoded, checksum.encoded, err
|
wtcs.encoded, checksum.encoded
|
||||||
);
|
);
|
||||||
return Err(Error::other(format!(
|
return Err(Error::other(format!(
|
||||||
"complete_multipart_upload checksum matches failed want={}, got={}",
|
"complete_multipart_upload checksum matches failed want={}, got={}",
|
||||||
|
|||||||
@@ -3507,10 +3507,6 @@ struct TransitionUploadedSaveProbeState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct TransitionUploadedSaveProbe {
|
struct TransitionUploadedSaveProbe {
|
||||||
state: Arc<TransitionUploadedSaveProbeState>,
|
state: Arc<TransitionUploadedSaveProbeState>,
|
||||||
}
|
}
|
||||||
@@ -3521,10 +3517,6 @@ static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Opti
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
impl TransitionUploadedSaveProbe {
|
impl TransitionUploadedSaveProbe {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn install(bucket: &str, object: &str) -> Self {
|
fn install(bucket: &str, object: &str) -> Self {
|
||||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
@@ -3541,10 +3533,6 @@ impl TransitionUploadedSaveProbe {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn attempts(&self) -> usize {
|
fn attempts(&self) -> usize {
|
||||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||||
}
|
}
|
||||||
@@ -3750,10 +3738,6 @@ struct TransitionCommitBarrierState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct TransitionCommitBarrier {
|
struct TransitionCommitBarrier {
|
||||||
state: Arc<TransitionCommitBarrierState>,
|
state: Arc<TransitionCommitBarrierState>,
|
||||||
}
|
}
|
||||||
@@ -3764,26 +3748,14 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Ar
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
impl TransitionCommitBarrier {
|
impl TransitionCommitBarrier {
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
|
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
|
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn install(bucket: &str, object: &str) -> Self {
|
fn install(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
|
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn install_after_lease_check(bucket: &str, object: &str) -> Self {
|
fn install_after_lease_check(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
|
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
|
||||||
}
|
}
|
||||||
@@ -3806,20 +3778,12 @@ impl TransitionCommitBarrier {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
async fn wait_until_paused(&self) {
|
async fn wait_until_paused(&self) {
|
||||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
.await
|
.await
|
||||||
.expect("transition should reach the deterministic commit barrier");
|
.expect("transition should reach the deterministic commit barrier");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
|
||||||
)]
|
|
||||||
fn release(&self) {
|
fn release(&self) {
|
||||||
self.state.release.notify_one();
|
self.state.release.notify_one();
|
||||||
}
|
}
|
||||||
@@ -5656,9 +5620,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
// TODO: Lifecycle
|
// TODO: Lifecycle
|
||||||
|
|
||||||
let mut version_found = true;
|
let mut version_found = true;
|
||||||
// delete_object_version below derives its own majority quorum from the
|
let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
||||||
// disk array, so the object-derived quorum here is unused.
|
|
||||||
let (mut goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
|
||||||
if let Some(err) = &gerr
|
if let Some(err) = &gerr
|
||||||
&& goi.name.is_empty()
|
&& goi.name.is_empty()
|
||||||
{
|
{
|
||||||
@@ -5845,14 +5807,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
|
|
||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||||
// MRF journal intent: partial-write recovery must survive a restart
|
|
||||||
// (HS-01); the heal request below remains the in-memory fast path.
|
|
||||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
|
||||||
rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
uuid::Uuid::try_parse(version_id).ok(),
|
|
||||||
);
|
|
||||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||||
bucket.to_string(),
|
bucket.to_string(),
|
||||||
Some(object.to_string()),
|
Some(object.to_string()),
|
||||||
@@ -6420,7 +6374,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||||
|
|
||||||
for disk in disks.iter() {
|
for disk in disks.iter() {
|
||||||
if disk.is_some() {
|
if let Some(disk) = disk {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let _ = self
|
let _ = self
|
||||||
@@ -7551,7 +7505,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT,
|
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT,
|
||||||
GetObjectFailureReason,
|
GetObjectFailureReason,
|
||||||
};
|
};
|
||||||
use crate::disk::RUSTFS_META_BUCKET;
|
use crate::disk::RUSTFS_META_BUCKET;
|
||||||
@@ -7683,8 +7637,8 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
legacy_completed,
|
legacy_completed,
|
||||||
internal_cancelled,
|
internal_cancelled,
|
||||||
legacy_cancelled,
|
legacy_cancelled,
|
||||||
internal_not_found_miss,
|
internal_unsafe_miss,
|
||||||
legacy_not_found_miss,
|
legacy_unsafe_miss,
|
||||||
internal_saved,
|
internal_saved,
|
||||||
legacy_saved,
|
legacy_saved,
|
||||||
) = metrics::with_local_recorder(&recorder, || {
|
) = metrics::with_local_recorder(&recorder, || {
|
||||||
@@ -7760,7 +7714,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_INTERNAL_META),
|
("path", GET_OBJECT_PATH_INTERNAL_META),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND),
|
("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
recorder.counter_value(
|
recorder.counter_value(
|
||||||
@@ -7768,7 +7722,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND),
|
("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -7819,21 +7773,21 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
"internal metadata lifecycle cancelled count must not leak into legacy_duplex"
|
"internal metadata lifecycle cancelled count must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
internal_not_found_miss, 1,
|
internal_unsafe_miss, 1,
|
||||||
"internal metadata not-found early-stop miss must retain its path label"
|
"internal metadata unsafe early-stop miss must retain its path label"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
legacy_not_found_miss, 0,
|
legacy_unsafe_miss, 0,
|
||||||
"internal metadata not-found early-stop miss must not leak into legacy_duplex"
|
"internal metadata unsafe early-stop miss must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
internal_saved,
|
internal_saved,
|
||||||
vec![0.0],
|
vec![0.0],
|
||||||
"internal metadata not-found miss must record zero saved responses on internal_meta"
|
"internal metadata unsafe miss must record zero saved responses on internal_meta"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
legacy_saved.is_empty(),
|
legacy_saved.is_empty(),
|
||||||
"internal metadata not-found miss saved responses must not leak into legacy_duplex"
|
"internal metadata unsafe miss saved responses must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -10216,288 +10170,6 @@ mod transition_upload_integrity_tests {
|
|||||||
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Compresses `plaintext` with the codec the PUT path uses, so the stored
|
|
||||||
/// bytes round-trip through the read path's decompressor.
|
|
||||||
async fn compress_for_storage(plaintext: &[u8]) -> Vec<u8> {
|
|
||||||
let mut reader = crate::io_support::rio::compression_reader(
|
|
||||||
Cursor::new(plaintext.to_vec()),
|
|
||||||
rustfs_utils::CompressionAlgorithm::default(),
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
let mut compressed = Vec::new();
|
|
||||||
reader.read_to_end(&mut compressed).await.expect("plaintext should compress");
|
|
||||||
assert!(compressed.len() < plaintext.len(), "test payload must actually compress");
|
|
||||||
compressed
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Writes a genuinely compressed object: stored data is `compressed`, and the
|
|
||||||
/// metadata marks it compressed with the plaintext length as its actual size,
|
|
||||||
/// exactly as the app-layer compress path records it.
|
|
||||||
async fn write_compressed_source(
|
|
||||||
set_disks: &Arc<SetDisks>,
|
|
||||||
disk_stores: &[DiskStore],
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
plaintext: &[u8],
|
|
||||||
compressed: &[u8],
|
|
||||||
) -> ObjectInfo {
|
|
||||||
for disk in disk_stores {
|
|
||||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
|
||||||
}
|
|
||||||
let mut user_defined = HashMap::new();
|
|
||||||
rustfs_utils::http::insert_str(
|
|
||||||
&mut user_defined,
|
|
||||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
|
||||||
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
|
||||||
);
|
|
||||||
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
|
|
||||||
let stream = crate::io_support::rio::HashReader::from_stream(
|
|
||||||
Cursor::new(compressed.to_vec()),
|
|
||||||
compressed.len() as i64,
|
|
||||||
plaintext.len() as i64,
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.expect("hash reader over compressed bytes");
|
|
||||||
let mut reader = PutObjReader::new(stream);
|
|
||||||
set_disks
|
|
||||||
.put_object(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&mut reader,
|
|
||||||
&ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
user_defined,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("compressed object should be written")
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_transitioned(
|
|
||||||
set_disks: &Arc<SetDisks>,
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
range: Option<HTTPRangeSpec>,
|
|
||||||
opts: &ObjectOptions,
|
|
||||||
) -> (Vec<u8>, i64) {
|
|
||||||
let mut reader = set_disks
|
|
||||||
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
|
|
||||||
.await
|
|
||||||
.expect("transitioned object reader should open");
|
|
||||||
let published_size = reader.object_info.size;
|
|
||||||
let mut body = Vec::new();
|
|
||||||
reader
|
|
||||||
.stream
|
|
||||||
.read_to_end(&mut body)
|
|
||||||
.await
|
|
||||||
.expect("transitioned body should drain");
|
|
||||||
(body, published_size)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Transition uploads the object's STORED bytes, so a tiered read has to
|
|
||||||
/// apply the same transform an erasure read would. #6107 routed this path
|
|
||||||
/// through `ReadPlan` to stop serving an encrypted object's ciphertext;
|
|
||||||
/// compression rides the same plan, and nothing pinned it (backlog#1851).
|
|
||||||
/// Without the transform this GET returns the compressed bytes under the
|
|
||||||
/// compressed size — silent corruption for every client of a compressed
|
|
||||||
/// object that ILM has moved to a warm tier.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn transitioned_compressed_object_get_returns_plaintext() {
|
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
|
||||||
let bucket = "transitioned-compressed-get-bucket";
|
|
||||||
let object = "object.txt";
|
|
||||||
let plaintext = b"transitioned compressed objects must decompress on read ".repeat(20_000);
|
|
||||||
let compressed = compress_for_storage(&plaintext).await;
|
|
||||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
|
||||||
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let (local_body, local_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
|
||||||
assert_eq!(local_body, plaintext, "control: the pre-transition read must decompress");
|
|
||||||
assert_eq!(
|
|
||||||
local_size,
|
|
||||||
plaintext.len() as i64,
|
|
||||||
"control: the pre-transition read publishes the plaintext size"
|
|
||||||
);
|
|
||||||
|
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
|
||||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
|
||||||
set_disks
|
|
||||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
|
||||||
.await
|
|
||||||
.expect("transition should commit");
|
|
||||||
|
|
||||||
let put_versions = backend.put_versions().await;
|
|
||||||
assert_eq!(put_versions.len(), 1, "transition should upload one remote candidate");
|
|
||||||
let remote_bytes = backend
|
|
||||||
.bytes(&put_versions[0].0)
|
|
||||||
.await
|
|
||||||
.expect("remote candidate should be stored");
|
|
||||||
assert_eq!(
|
|
||||||
remote_bytes, compressed,
|
|
||||||
"transition uploads the stored representation; the read side is what has to decode it"
|
|
||||||
);
|
|
||||||
|
|
||||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
|
||||||
assert_eq!(body, plaintext, "a tiered read must return the object's content, not its stored bytes");
|
|
||||||
assert_eq!(
|
|
||||||
published_size,
|
|
||||||
plaintext.len() as i64,
|
|
||||||
"a tiered read must publish the plaintext size, not the compressed one"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A ranged tiered read is expressed in plaintext coordinates, so the plan
|
|
||||||
/// has to translate it into the remote copy's compressed extent and skip
|
|
||||||
/// into the decompressed stream — the same translation the erasure path does.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn transitioned_compressed_object_range_get_returns_plaintext_slice() {
|
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
|
||||||
let bucket = "transitioned-compressed-range-bucket";
|
|
||||||
let object = "object.txt";
|
|
||||||
let plaintext = b"ranged reads of transitioned compressed objects must land in plaintext ".repeat(20_000);
|
|
||||||
let compressed = compress_for_storage(&plaintext).await;
|
|
||||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
|
||||||
|
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
|
||||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
|
||||||
set_disks
|
|
||||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
|
||||||
.await
|
|
||||||
.expect("transition should commit");
|
|
||||||
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
// Deliberately past the compressed size, so a range still measured in
|
|
||||||
// stored coordinates could not produce this slice.
|
|
||||||
let start = compressed.len() as i64 + 4096;
|
|
||||||
let end = start + 511;
|
|
||||||
let range = HTTPRangeSpec {
|
|
||||||
is_suffix_length: false,
|
|
||||||
start,
|
|
||||||
end,
|
|
||||||
};
|
|
||||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
|
||||||
|
|
||||||
let expected = &plaintext[start as usize..=end as usize];
|
|
||||||
assert_eq!(body, expected, "a ranged tiered read must return that plaintext slice");
|
|
||||||
assert_eq!(published_size, expected.len() as i64, "a ranged tiered read publishes the slice length");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The restore copy-back re-writes the object under its original metadata,
|
|
||||||
/// which still says "compressed". It therefore has to keep receiving the
|
|
||||||
/// STORED bytes: `restore_request_active` holds it on the plan's `Plain`
|
|
||||||
/// branch, and decompressing there would write plaintext under compressed
|
|
||||||
/// metadata.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn restore_read_of_transitioned_compressed_object_keeps_stored_bytes() {
|
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
|
||||||
let bucket = "transitioned-compressed-restore-bucket";
|
|
||||||
let object = "object.txt";
|
|
||||||
let plaintext = b"restore copy-back must keep the stored representation intact ".repeat(20_000);
|
|
||||||
let compressed = compress_for_storage(&plaintext).await;
|
|
||||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
|
||||||
|
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
|
||||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
|
||||||
set_disks
|
|
||||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
|
||||||
.await
|
|
||||||
.expect("transition should commit");
|
|
||||||
|
|
||||||
let oi = set_disks
|
|
||||||
.get_object_info(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("transitioned metadata should resolve");
|
|
||||||
let restore_opts = ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
part_number: Some(1),
|
|
||||||
transition: TransitionOptions {
|
|
||||||
restore_request: s3s::dto::RestoreRequest {
|
|
||||||
days: Some(1),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
&None,
|
|
||||||
&HeaderMap::new(),
|
|
||||||
&oi,
|
|
||||||
&restore_opts,
|
|
||||||
&set_disks.ctx.tier_config_mgr(),
|
|
||||||
set_disks.ctx.object_encryption_resolver(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.expect("restore read of the tiered copy should open");
|
|
||||||
let published_size = reader.object_info.size;
|
|
||||||
let mut body = Vec::new();
|
|
||||||
reader.stream.read_to_end(&mut body).await.expect("restore body should drain");
|
|
||||||
|
|
||||||
assert_eq!(body, compressed, "a restore read must copy the stored bytes back verbatim");
|
|
||||||
assert_eq!(
|
|
||||||
published_size,
|
|
||||||
compressed.len() as i64,
|
|
||||||
"a restore read must keep publishing the stored size"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Plain objects must keep streaming the remote bytes through untouched:
|
|
||||||
/// their plan is `Plain`, so the tiered read stays byte-identical.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn transitioned_plain_object_get_is_unchanged() {
|
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
|
||||||
let bucket = "transitioned-plain-get-bucket";
|
|
||||||
let object = "object.bin";
|
|
||||||
let payload = b"plain transitioned objects must keep reading back byte-identical ".repeat(1024);
|
|
||||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
|
||||||
|
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
|
||||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
|
||||||
set_disks
|
|
||||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
|
||||||
.await
|
|
||||||
.expect("transition should commit");
|
|
||||||
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
|
||||||
assert_eq!(body, payload);
|
|
||||||
assert_eq!(published_size, payload.len() as i64);
|
|
||||||
|
|
||||||
let range = HTTPRangeSpec {
|
|
||||||
is_suffix_length: false,
|
|
||||||
start: 100,
|
|
||||||
end: 611,
|
|
||||||
};
|
|
||||||
let (ranged_body, ranged_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
|
||||||
assert_eq!(ranged_body, &payload[100..=611]);
|
|
||||||
assert_eq!(ranged_size, payload.len() as i64, "a plain ranged read keeps publishing the object size");
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn corrupt_beyond_read_quorum(
|
async fn corrupt_beyond_read_quorum(
|
||||||
temp_dirs: &[tempfile::TempDir],
|
temp_dirs: &[tempfile::TempDir],
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -116,7 +116,6 @@ impl SetDisks {
|
|||||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
||||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||||
MetadataCacheLookup::Hit(entry) => Some(entry),
|
MetadataCacheLookup::Hit(entry) => Some(entry),
|
||||||
@@ -1077,15 +1076,6 @@ impl SetDisks {
|
|||||||
"Recoverable decode error triggered read repair"
|
"Recoverable decode error triggered read repair"
|
||||||
);
|
);
|
||||||
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
||||||
// MRF journal intent: keeps a durable Urgent ECDecode
|
|
||||||
// request alive across restarts even when the in-memory
|
|
||||||
// read-repair request is dropped or lost (HS-01).
|
|
||||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
|
||||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
fi.version_id,
|
|
||||||
);
|
|
||||||
submit_read_repair_heal(
|
submit_read_repair_heal(
|
||||||
bucket,
|
bucket,
|
||||||
object,
|
object,
|
||||||
@@ -1836,7 +1826,6 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
|
|||||||
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
|
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
|
||||||
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
|
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
|
||||||
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
|
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
|
||||||
}
|
}
|
||||||
@@ -3897,15 +3886,13 @@ mod tests {
|
|||||||
assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
|
assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
|
||||||
// observe=false (non-observed fanout) also disables early-stop.
|
// observe=false (non-observed fanout) also disables early-stop.
|
||||||
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
|
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
|
||||||
// Whole/latest data-read metadata is now allowed by default;
|
assert!(!metadata_early_stop_permitted(true, true, true, "", false, false));
|
||||||
// the inline verifier still decides whether it can stop early.
|
|
||||||
assert!(metadata_early_stop_permitted(true, true, true, "", false, false));
|
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn metadata_early_stop_allows_safe_data_reads_by_default() {
|
fn metadata_early_stop_keeps_data_reads_opt_in_by_default() {
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
[
|
[
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||||
@@ -3913,7 +3900,7 @@ mod tests {
|
|||||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
||||||
],
|
],
|
||||||
|| {
|
|| {
|
||||||
assert!(should_allow_metadata_early_stop(true, "", false, false));
|
assert!(!should_allow_metadata_early_stop(true, "", false, false));
|
||||||
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
||||||
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
||||||
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
|
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
|
||||||
@@ -3945,34 +3932,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn metadata_early_stop_bounded_fanout_defaults_to_enabled() {
|
|
||||||
temp_env::with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, None),
|
|
||||||
],
|
|
||||||
|| {
|
|
||||||
assert!(is_get_metadata_data_read_early_stop_enabled());
|
|
||||||
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
|
||||||
},
|
|
||||||
);
|
|
||||||
temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false"))], || {
|
|
||||||
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
|
|
||||||
});
|
|
||||||
temp_env::with_vars(
|
|
||||||
[
|
|
||||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
|
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
|
|
||||||
],
|
|
||||||
|| {
|
|
||||||
assert!(!is_get_metadata_data_read_early_stop_enabled());
|
|
||||||
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn metadata_early_stop_rejects_healing_and_free_version_requests() {
|
fn metadata_early_stop_rejects_healing_and_free_version_requests() {
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ use tracing::trace;
|
|||||||
|
|
||||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||||
const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
|
|
||||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||||
|
|
||||||
@@ -257,40 +256,13 @@ impl ECStore {
|
|||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||||
let object = encode_dir_object(object);
|
let _ = (bucket, object, opts);
|
||||||
let pools = self.get_pools_for_heal_object(opts)?;
|
// Stale multipart reconciliation is already owned by the lifecycle-driven
|
||||||
|
// background cleanup path in `bucket_lifecycle_ops.rs`. There is currently
|
||||||
let mut futures = Vec::with_capacity(pools.len());
|
// no stable object-heal contract that should fan this request out through
|
||||||
for pool in pools.iter() {
|
// pool/set storage layers, so keep the placeholder explicit at the ECStore
|
||||||
futures.push(pool.check_abandoned_parts(bucket, &object, opts));
|
// boundary instead of dispatching into lower layers.
|
||||||
}
|
Err(StorageError::NotImplemented)
|
||||||
|
|
||||||
let mut first_error = None;
|
|
||||||
for result in join_all(futures).await {
|
|
||||||
if let Err(err) = result
|
|
||||||
&& first_error.is_none()
|
|
||||||
{
|
|
||||||
first_error = Some(err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(err) = first_error {
|
|
||||||
return Err(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
trace!(
|
|
||||||
event = EVENT_HEAL_ABANDONED_PARTS,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
|
||||||
state = "completed",
|
|
||||||
result = "ok",
|
|
||||||
bucket,
|
|
||||||
object,
|
|
||||||
dry_run = opts.dry_run,
|
|
||||||
"Heal abandoned parts completed"
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user