mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 10:43:15 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ce939a8117 |
@@ -66,8 +66,8 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||
@echo "📣 Checking FIPS wording guard..."
|
||||
./scripts/check_fips_wording.sh
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
|
||||
@@ -117,9 +117,6 @@ jobs:
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -152,9 +152,6 @@ jobs:
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -15,35 +15,28 @@
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
||||
# and uploads them to Cloudflare R2.
|
||||
#
|
||||
# Trigger:
|
||||
# - workflow_run: automatically package after "Build and Release" completes
|
||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
||||
# to the GitHub release before packaging starts)
|
||||
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
||||
# - release published: automatically package when a GitHub release is published
|
||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||
#
|
||||
# Flow:
|
||||
# 1. Resolve the triggering Build workflow run for the release tag
|
||||
# 1. Find the Build workflow run for the release tag
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64
|
||||
# 4. Build RPM packages for x86_64 and aarch64
|
||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
||||
# 5. Upload all packages to Cloudflare R2
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
# contents: write is required to upload packages to the GitHub release
|
||||
contents: write
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
on:
|
||||
# Follows the same pattern as docker.yml: run after the release build
|
||||
# workflow completes, so packaging is triggered only by release tags
|
||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
release:
|
||||
types: [ published ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -56,26 +49,13 @@ on:
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
||||
# Tag pushes arrive as event == push with head_branch != main (a
|
||||
# non-main push head_branch is the release tag name). Manual dispatch
|
||||
# stays available as a fallback for backfills and re-runs.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -95,8 +75,8 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
# Determine tag
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
TAG="${HEAD_BRANCH}"
|
||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||
TAG="${{ github.event.release.tag_name }}"
|
||||
elif [[ -n "$INPUT_TAG" ]]; then
|
||||
TAG="$INPUT_TAG"
|
||||
else
|
||||
@@ -113,11 +93,6 @@ jobs:
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Use the Build and Release run that triggered this workflow
|
||||
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
||||
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
# Find the build run that produced this tag
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
@@ -481,54 +456,6 @@ jobs:
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TAG="${{ needs.resolve.outputs.tag }}"
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||
|
||||
# Upload the packages, then refresh the release checksums so the new
|
||||
# assets are covered, matching the binary release flow.
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
||||
gh release upload "$TAG" "$f" --clobber
|
||||
fi
|
||||
done
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
touch "$checksum_file"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
# Remove any stale entry, then append the fresh digest
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
mv "${checksum_file}.tmp" "$checksum_file"
|
||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
||||
gh release upload "$TAG" "$checksum_file" --clobber
|
||||
done
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
+4
-4
@@ -31,7 +31,7 @@ HTTP request
|
||||
→ storage/ecfs (erasure coding, encryption, checksums)
|
||||
→ ecstore (disk pool selection, data distribution)
|
||||
→ rio (reader pipeline: encrypt → compress → hash → write)
|
||||
→ io-core (buffer pool, storage profiling, admission control)
|
||||
→ io-core (zero-copy I/O, buffer pool, direct I/O)
|
||||
→ local disk / remote disk via RPC
|
||||
```
|
||||
|
||||
@@ -55,7 +55,7 @@ rustfs/ # Workspace root (virtual manifest)
|
||||
├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members)
|
||||
│ ├── ecstore/ # Erasure-coded storage engine
|
||||
│ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash)
|
||||
│ ├── io-core/ # Buffer pool, storage profiling, admission control
|
||||
│ ├── io-core/ # Zero-copy I/O, scheduling, buffer pool
|
||||
│ ├── io-metrics/ # I/O metrics collection
|
||||
│ ├── common/ # Shared runtime state, globals, data usage types
|
||||
│ ├── config/ # Configuration types and parsing
|
||||
@@ -302,7 +302,7 @@ The binary (`main.rs`) boots in this order:
|
||||
│ │ │
|
||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (core) │ │ (readers) │ │ (buffers) │
|
||||
│ (core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
@@ -314,7 +314,7 @@ The binary (`main.rs`) boots in this order:
|
||||
|
||||
- **"Where does S3 PutObject go?"**
|
||||
`server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
|
||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` supplies buffers
|
||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
|
||||
|
||||
- **"Where are bucket policies enforced?"**
|
||||
`app/bucket_usecase` calls into `crates/policy/`
|
||||
|
||||
Generated
+104
-114
@@ -964,9 +964,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-kms"
|
||||
version = "1.115.0"
|
||||
version = "1.114.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
|
||||
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -990,9 +990,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.142.0"
|
||||
version = "1.141.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
|
||||
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1027,9 +1027,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.106.0"
|
||||
version = "1.105.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
|
||||
checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1053,9 +1053,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.108.0"
|
||||
version = "1.107.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
|
||||
checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1079,9 +1079,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.111.0"
|
||||
version = "1.110.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
|
||||
checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1598,7 +1598,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1617,7 +1617,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1858,9 +1858,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.3"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
||||
checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -1968,7 +1968,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2428,7 +2428,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2453,11 +2453,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3664,7 +3664,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3924,7 +3924,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4148,9 +4148,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.11"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
|
||||
checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de"
|
||||
|
||||
[[package]]
|
||||
name = "findshlibs"
|
||||
@@ -4369,9 +4369,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4380,11 +4380,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "1.4.5"
|
||||
version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -4726,9 +4726,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.16"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
||||
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -5028,9 +5028,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.23.3"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
|
||||
checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5062,9 +5062,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.23.3"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
|
||||
checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5073,15 +5073,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros-meta"
|
||||
version = "0.23.3"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
|
||||
checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-meta"
|
||||
version = "0.23.3"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
|
||||
checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f"
|
||||
dependencies = [
|
||||
"hotpath-macros-meta",
|
||||
]
|
||||
@@ -5280,9 +5280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
|
||||
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"potential_utf",
|
||||
@@ -5294,9 +5294,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_core"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
|
||||
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
@@ -5307,9 +5307,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
|
||||
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
@@ -5321,17 +5321,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
|
||||
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
|
||||
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
"icu_properties_data",
|
||||
@@ -5342,15 +5341,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
|
||||
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "2.3.0"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
|
||||
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locale_core",
|
||||
@@ -5418,7 +5417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5969,9 +5968,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.20"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a"
|
||||
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -6034,9 +6033,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.3"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||
|
||||
[[package]]
|
||||
name = "local-ip-address"
|
||||
@@ -6483,9 +6482,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mqttbytes-core-next"
|
||||
version = "0.34.0"
|
||||
version = "0.33.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa"
|
||||
checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"thiserror 2.0.20",
|
||||
@@ -6886,7 +6885,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7345,9 +7344,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pageant"
|
||||
version = "0.2.2"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
|
||||
checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"byteorder",
|
||||
@@ -7729,9 +7728,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
@@ -7837,9 +7836,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.6"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
|
||||
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
|
||||
dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
@@ -8047,7 +8046,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -8067,7 +8066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -8088,7 +8087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8101,7 +8100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8217,7 +8216,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"spin 0.12.3",
|
||||
"spin 0.12.2",
|
||||
"symbolic-demangle",
|
||||
"tempfile",
|
||||
"thiserror 2.0.20",
|
||||
@@ -8286,9 +8285,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.17"
|
||||
version = "0.11.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"
|
||||
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
@@ -8554,9 +8553,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.6.0"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
||||
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -8568,7 +8567,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"itoa",
|
||||
"num-bigint 0.5.1",
|
||||
"num-bigint 0.4.8",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
@@ -8869,9 +8868,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-core-next"
|
||||
version = "0.34.0"
|
||||
version = "0.33.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc"
|
||||
checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c"
|
||||
dependencies = [
|
||||
"async-tungstenite",
|
||||
"futures-io",
|
||||
@@ -8887,18 +8886,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-next"
|
||||
version = "0.34.0"
|
||||
version = "0.33.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "477c9bbfba8f3aecc7aad31c6de2eacb75822efaa18e7aeecb8d3d8e534fbf07"
|
||||
checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08"
|
||||
dependencies = [
|
||||
"rumqttc-v5-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-v5-next"
|
||||
version = "0.34.0"
|
||||
version = "0.33.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf"
|
||||
checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc"
|
||||
dependencies = [
|
||||
"async-tungstenite",
|
||||
"bytes",
|
||||
@@ -8921,9 +8920,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.7"
|
||||
version = "0.62.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
|
||||
checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5"
|
||||
dependencies = [
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
@@ -8946,7 +8945,7 @@ dependencies = [
|
||||
"enum_dispatch",
|
||||
"flate2",
|
||||
"futures",
|
||||
"generic-array 1.4.5",
|
||||
"generic-array 1.4.4",
|
||||
"getrandom 0.4.3",
|
||||
"ghash",
|
||||
"hex-literal",
|
||||
@@ -9281,7 +9280,6 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
@@ -9465,7 +9463,6 @@ dependencies = [
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
@@ -9493,7 +9490,7 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"rayon",
|
||||
"smallvec",
|
||||
"spin 0.12.3",
|
||||
"spin 0.12.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9539,8 +9536,6 @@ version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"futures",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -9613,6 +9608,7 @@ version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
"memmap2",
|
||||
"rustfs-io-metrics",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
@@ -9829,19 +9825,14 @@ name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"humantime",
|
||||
"hyper",
|
||||
"jiff",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10255,7 +10246,6 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-lock",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
@@ -10836,7 +10826,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11400,9 +11390,9 @@ checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.12.3"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10"
|
||||
checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
@@ -11814,7 +11804,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -11978,9 +11968,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.4"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
|
||||
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
@@ -12654,9 +12644,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.24.1"
|
||||
version = "1.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
|
||||
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
@@ -13171,9 +13161,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.4"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
||||
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||
|
||||
[[package]]
|
||||
name = "x509-cert"
|
||||
@@ -13353,9 +13343,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerotrie"
|
||||
version = "0.2.5"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
|
||||
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
@@ -13364,9 +13354,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.7"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
|
||||
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
@@ -13375,13 +13365,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.11.5"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
|
||||
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+8
-8
@@ -228,9 +228,9 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
@@ -284,8 +284,8 @@ rayon = "1.12.0"
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
redis = { version = "1.5.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
@@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.24.1" }
|
||||
uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.7" }
|
||||
russh = { version = "0.62.6" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -350,7 +350,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -40,7 +40,6 @@ mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
thr = "thr"
|
||||
mis = "mis"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
@@ -42,7 +42,6 @@ chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -224,13 +224,6 @@ pub struct HealOpts {
|
||||
pub enum HealAdmissionDropReason {
|
||||
QueueFull,
|
||||
PolicyDropped,
|
||||
/// HS-06: an admin heal start overlaps (same bucket with mutually
|
||||
/// containing prefixes, or the same erasure set) an already running or
|
||||
/// queued task. Only produced when RUSTFS_HEAL_OVERLAP_POLICY=minio_error.
|
||||
AlreadyRunning,
|
||||
/// HS-06: same as [`Self::AlreadyRunning`] but for paths that merely
|
||||
/// contain (or are contained by) the active task's path.
|
||||
OverlappingPaths,
|
||||
}
|
||||
|
||||
impl HealAdmissionDropReason {
|
||||
@@ -238,8 +231,6 @@ impl HealAdmissionDropReason {
|
||||
match self {
|
||||
Self::QueueFull => "queue_full",
|
||||
Self::PolicyDropped => "policy_dropped",
|
||||
Self::AlreadyRunning => "already_running",
|
||||
Self::OverlappingPaths => "overlapping_paths",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -296,9 +287,6 @@ pub enum HealRequestSource {
|
||||
Scanner,
|
||||
AutoHeal,
|
||||
ReadRepair,
|
||||
/// Mission Repair Feed: intents delivered by error paths and replayed
|
||||
/// from the durable MRF journal.
|
||||
Mrf,
|
||||
}
|
||||
|
||||
impl HealRequestSource {
|
||||
@@ -309,7 +297,6 @@ impl HealRequestSource {
|
||||
Self::Scanner => "scanner",
|
||||
Self::AutoHeal => "auto_heal",
|
||||
Self::ReadRepair => "read_repair",
|
||||
Self::Mrf => "mrf",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -326,9 +313,6 @@ pub enum HealChannelCommand {
|
||||
Query {
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
/// Incremental result cursor (HS-06): only items with a sequence
|
||||
/// greater than this are returned; `None` keeps the full snapshot.
|
||||
since_seq: Option<u64>,
|
||||
response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
|
||||
},
|
||||
/// Cancel heal task
|
||||
@@ -534,21 +518,10 @@ async fn receive_heal_channel_response(
|
||||
|
||||
/// Send heal query request
|
||||
pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
|
||||
query_heal_status_since(heal_path, client_token, None).await
|
||||
}
|
||||
|
||||
/// Incremental heal query (HS-06): pass the client's last seen sequence
|
||||
/// number to receive only newer result items.
|
||||
pub async fn query_heal_status_since(
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<HealChannelResponse, String> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
send_heal_command(HealChannelCommand::Query {
|
||||
heal_path,
|
||||
client_token,
|
||||
since_seq,
|
||||
response_tx,
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -17,10 +17,8 @@ pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
pub mod mrf_channel;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
pub mod trace_bus;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Mission Repair Feed (MRF) intent channel.
|
||||
//!
|
||||
//! Producers on error paths (read decode failure, scanner metadata
|
||||
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
||||
//! heal crate through a global bounded channel. Delivery is strictly
|
||||
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
||||
//! (counting it) when the channel is full or uninitialized — losing one heal
|
||||
//! hint is always preferred over stalling an IO path. Durable replay of
|
||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
||||
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
||||
/// dropping (and counting) intents, never by blocking the producer.
|
||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
||||
|
||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
||||
/// PartialWrite -> Normal).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfKind {
|
||||
/// Erasure decode failed while serving a read (read path).
|
||||
DecodeFailure,
|
||||
/// Scanner classified object metadata as corrupt.
|
||||
MetadataCorruption,
|
||||
/// A write left the object with fewer committed shards than the set size.
|
||||
PartialWrite,
|
||||
}
|
||||
|
||||
impl MrfKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
MrfKind::DecodeFailure => "decode-failure",
|
||||
MrfKind::MetadataCorruption => "metadata-corruption",
|
||||
MrfKind::PartialWrite => "partial-write",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
||||
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
||||
/// intent never re-allocates the strings.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MrfIntent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
/// Version the intent targets, as raw UUID bytes.
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
pub kind: MrfKind,
|
||||
pub enqueued_at_ms: u64,
|
||||
/// Times this intent has already been offered to the heal manager.
|
||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
||||
pub attempts: u8,
|
||||
}
|
||||
|
||||
/// Consumer-side retry ceiling before an intent is given up on.
|
||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
||||
|
||||
impl MrfIntent {
|
||||
/// Rough in-memory footprint used by the queue's byte budget.
|
||||
pub fn estimated_bytes(&self) -> usize {
|
||||
// Struct + strings + version bytes; buckets and objects are usually
|
||||
// far below this bound, so rounding up keeps the budget conservative.
|
||||
64 + self.bucket.len() + self.object.len()
|
||||
}
|
||||
}
|
||||
|
||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
||||
|
||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
||||
/// this before touching the channel so the disabled path stays allocation- and
|
||||
/// sync-free.
|
||||
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
||||
|
||||
/// Override delivery (used at heal-runtime startup from configuration).
|
||||
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
||||
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether producers currently deliver intents.
|
||||
pub fn mrf_delivery_enabled() -> bool {
|
||||
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Create the global MRF channel and return the consumer half. Fails if the
|
||||
/// channel is already initialized (the heal runtime is a singleton).
|
||||
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
||||
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
||||
GLOBAL_MRF_SENDER
|
||||
.set(sender)
|
||||
.map_err(|_| "MRF channel sender already initialized")?;
|
||||
Ok(receiver)
|
||||
}
|
||||
|
||||
/// Best-effort, non-blocking intent delivery from an error path.
|
||||
///
|
||||
/// Returns `true` when the intent was accepted into the channel. `false`
|
||||
/// means the intent was dropped (feature disabled, channel not yet
|
||||
/// initialized, or channel full) — callers must not retry or await; the
|
||||
/// existing read-repair / scanner heal paths remain the safety net.
|
||||
///
|
||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
||||
if !mrf_delivery_enabled() {
|
||||
return false;
|
||||
}
|
||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
||||
return false;
|
||||
};
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
||||
kind,
|
||||
enqueued_at_ms: unix_now_ms(),
|
||||
attempts: 0,
|
||||
};
|
||||
sender.try_send(intent).is_ok()
|
||||
}
|
||||
|
||||
fn unix_now_ms() -> u64 {
|
||||
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
||||
// failure would be a bug rather than something to handle here.
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn intents_estimate_is_conservative() {
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from("bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: Some([0u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
};
|
||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
||||
|
||||
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
||||
let intent = receiver.recv().await.expect("intent should arrive");
|
||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(intent.bucket.as_ref(), "b");
|
||||
|
||||
// Disable delivery: producers become no-ops.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
|
||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
||||
// never blocking.
|
||||
let mut accepted = 0;
|
||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
||||
accepted += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_send_without_channel_is_false() {
|
||||
// This test may run after the tokio test above in the same process;
|
||||
// the singleton semantics make a clean "uninitialized" case hard, so
|
||||
// assert the flag-off behavior only.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
||||
|
||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceKind {
|
||||
Heal,
|
||||
Scanner,
|
||||
}
|
||||
|
||||
impl TraceKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Heal => "heal",
|
||||
Self::Scanner => "scanner",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceFunc {
|
||||
HealTask,
|
||||
HealBucket,
|
||||
HealObject,
|
||||
HealCheckAbandonedParts,
|
||||
HealErasureSetPage,
|
||||
ScannerFolder,
|
||||
ScannerIlmAction,
|
||||
ScannerHealCandidate,
|
||||
Dropped,
|
||||
}
|
||||
|
||||
impl TraceFunc {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HealTask => "heal.Task",
|
||||
Self::HealBucket => "heal.Bucket",
|
||||
Self::HealObject => "heal.Object",
|
||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
||||
Self::ScannerFolder => "scanner.Folder",
|
||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
||||
Self::Dropped => "trace.Dropped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TraceVal {
|
||||
Bool(bool),
|
||||
U64(u64),
|
||||
I64(i64),
|
||||
Str(Arc<str>),
|
||||
}
|
||||
|
||||
impl From<bool> for TraceVal {
|
||||
fn from(value: bool) -> Self {
|
||||
Self::Bool(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for TraceVal {
|
||||
fn from(value: u64) -> Self {
|
||||
Self::U64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for TraceVal {
|
||||
fn from(value: i64) -> Self {
|
||||
Self::I64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for TraceVal {
|
||||
fn from(value: &str) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for TraceVal {
|
||||
fn from(value: String) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceAttr {
|
||||
pub key: &'static str,
|
||||
pub value: TraceVal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceEvent {
|
||||
pub kind: TraceKind,
|
||||
pub func: TraceFunc,
|
||||
pub time: SystemTime,
|
||||
pub bucket: Option<Arc<str>>,
|
||||
pub object: Option<Arc<str>>,
|
||||
pub duration: Duration,
|
||||
pub bytes: u64,
|
||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
||||
}
|
||||
|
||||
impl TraceEvent {
|
||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
func,
|
||||
time: SystemTime::now(),
|
||||
bucket: None,
|
||||
object: None,
|
||||
duration: Duration::ZERO,
|
||||
bytes: 0,
|
||||
attrs: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
||||
self.bucket = Some(bucket.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
||||
self.object = Some(object.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
||||
self.duration = duration;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
||||
self.bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
||||
self.attrs.push(TraceAttr {
|
||||
key,
|
||||
value: value.into(),
|
||||
});
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceBus {
|
||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let capacity = capacity.max(1);
|
||||
let (sender, _receiver) = broadcast::channel(capacity);
|
||||
Self {
|
||||
sender,
|
||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscriber_count(&self) -> usize {
|
||||
self.subscriber_count.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> TraceSubscription {
|
||||
let receiver = self.sender.subscribe();
|
||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
||||
TraceSubscription {
|
||||
receiver,
|
||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
if self.subscriber_count() == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.sender.send(Arc::new(build())).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TraceBus {
|
||||
fn default() -> Self {
|
||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceSubscription {
|
||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceSubscription {
|
||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
|
||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
||||
self.receiver.try_recv()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceSubscription {
|
||||
fn drop(&mut self) {
|
||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn global_trace_bus() -> &'static TraceBus {
|
||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
||||
}
|
||||
|
||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
||||
global_trace_bus().subscribe()
|
||||
}
|
||||
|
||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
global_trace_bus().emit(build)
|
||||
}
|
||||
|
||||
pub fn trace_subscriber_count() -> usize {
|
||||
global_trace_bus().subscriber_count()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
#[test]
|
||||
fn trace_emit_skips_builder_without_subscribers() {
|
||||
let bus = TraceBus::new(4);
|
||||
let built = AtomicUsize::new(0);
|
||||
|
||||
let sent = bus.emit(|| {
|
||||
built.fetch_add(1, Ordering::Relaxed);
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
});
|
||||
|
||||
assert!(!sent);
|
||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trace_subscriber_receives_event() {
|
||||
let bus = TraceBus::new(4);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
assert!(bus.emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("bucket")
|
||||
.with_object("object")
|
||||
.with_duration(Duration::from_millis(7))
|
||||
.with_bytes(11)
|
||||
.with_attr("dry", true)
|
||||
}));
|
||||
|
||||
let event = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect("subscriber should receive emitted trace event");
|
||||
|
||||
assert_eq!(event.kind, TraceKind::Heal);
|
||||
assert_eq!(event.func, TraceFunc::HealObject);
|
||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(event.object.as_deref(), Some("object"));
|
||||
assert_eq!(event.duration, Duration::from_millis(7));
|
||||
assert_eq!(event.bytes, 11);
|
||||
assert_eq!(
|
||||
event.attrs.as_slice(),
|
||||
&[TraceAttr {
|
||||
key: "dry",
|
||||
value: TraceVal::Bool(true)
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_subscription_drop_decrements_count() {
|
||||
let bus = TraceBus::new(4);
|
||||
let subscription = bus.subscribe();
|
||||
|
||||
assert_eq!(bus.subscriber_count(), 1);
|
||||
drop(subscription);
|
||||
assert_eq!(bus.subscriber_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
||||
let bus = TraceBus::new(2);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
for index in 0_u64..4 {
|
||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
||||
}
|
||||
|
||||
let err = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,9 @@
|
||||
|
||||
//! Shared backpressure policy type.
|
||||
//!
|
||||
//! This module only carries the watermark policy; the admission primitive it
|
||||
//! projects into lives in `rustfs-io-core`.
|
||||
//! The runtime backpressure implementation (byte-watermark pipes and
|
||||
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
||||
//! carries the watermark policy type that implementation shares.
|
||||
|
||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||
|
||||
|
||||
@@ -177,40 +177,3 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
||||
|
||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||
|
||||
/// Environment variable that toggles the MRF (mission repair feed) intent
|
||||
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
||||
/// unconsumed intents are replayed from the durable journal after a restart.
|
||||
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
||||
|
||||
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
||||
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
||||
|
||||
/// Environment variable for the MRF journal byte budget. The journal is
|
||||
/// compacted once its on-disk size crosses this bound.
|
||||
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
||||
|
||||
/// Environment variable for the MRF journal replay batch size (intents per
|
||||
/// replay push round).
|
||||
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
||||
|
||||
/// Default behavior keeps the MRF intent pipeline enabled.
|
||||
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
||||
|
||||
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
||||
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
||||
|
||||
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
||||
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Default MRF replay batch size.
|
||||
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
||||
|
||||
/// Environment variable selecting how admin heal starts behave when the
|
||||
/// requested path overlaps an already running or queued heal: `merge`
|
||||
/// (default, keep today's dedup/merge semantics) or `minio_error` (return a
|
||||
/// typed already-running / overlapping-paths rejection like madmin).
|
||||
pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY";
|
||||
|
||||
/// Default overlap policy: merge duplicate/overlapping requests.
|
||||
pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge";
|
||||
|
||||
@@ -234,31 +234,6 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||
|
||||
/// Enable foreground PutObject request admission.
|
||||
///
|
||||
/// This is an experimental, default-off foreground write backpressure gate for
|
||||
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
||||
/// path and only the existing request counters are updated.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
||||
|
||||
/// Maximum foreground PutObject requests admitted concurrently per process.
|
||||
///
|
||||
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
||||
/// A value of `0` disables the gate even when the enable flag is present, so a
|
||||
/// partially configured rollout cannot reject every PUT.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
||||
|
||||
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
||||
///
|
||||
/// Once this timeout expires the request fails before body ingest/storage
|
||||
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
||||
|
||||
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
||||
|
||||
/// Environment variable for minimum GetObject timeout in seconds.
|
||||
///
|
||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||
|
||||
@@ -870,157 +870,6 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
||||
/// scanner's writer-side cache has the same map type).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or is
|
||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
||||
/// was cut (largest first).
|
||||
pub fn prefix_usage_in_cache(
|
||||
cache: &HashMap<String, DataUsageEntry>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> Option<PrefixUsageQuery> {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
let root = if prefix.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
let child_prefix = format!("{root}/");
|
||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
||||
.children
|
||||
.iter()
|
||||
.filter_map(|child_key| {
|
||||
let child = cache.get(child_key)?;
|
||||
let child_flat = flatten_entry(cache, child, 1)?;
|
||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
||||
// slash marks a directory object and is display-only here.
|
||||
let name = child_key
|
||||
.strip_prefix(child_prefix.as_str())
|
||||
.unwrap_or(child_key.as_str())
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
Some(PrefixUsageEntry {
|
||||
prefix: name,
|
||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
sub_prefixes.sort_by(|left, right| {
|
||||
right
|
||||
.usage
|
||||
.size
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
let truncated = sub_prefixes.len() > max_entries;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
|
||||
Some(PrefixUsageQuery {
|
||||
usage,
|
||||
compacted: entry.compacted,
|
||||
truncated,
|
||||
sub_prefixes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
||||
|
||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
||||
return None;
|
||||
}
|
||||
let mut flattened = DataUsageEntry::default();
|
||||
if !flattened.checked_merge(root) {
|
||||
return None;
|
||||
}
|
||||
flattened.compacted = root.compacted;
|
||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
||||
// child edge pointing back at the root's own key is still terminated by
|
||||
// the visited set on first encounter.
|
||||
let mut visited: HashSet<&str> = HashSet::new();
|
||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
||||
while let Some((key, child_depth)) = pending.pop() {
|
||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let entry = cache.get(key)?;
|
||||
if !flattened.checked_merge(entry) {
|
||||
return None;
|
||||
}
|
||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
||||
}
|
||||
flattened.children.clear();
|
||||
Some(flattened)
|
||||
}
|
||||
|
||||
/// Flattened counters of one prefix subtree, as returned by
|
||||
/// [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageSummary {
|
||||
pub size: u64,
|
||||
pub objects: u64,
|
||||
pub versions: u64,
|
||||
pub delete_markers: u64,
|
||||
}
|
||||
|
||||
impl PrefixUsageSummary {
|
||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
||||
Self {
|
||||
size: entry.size as u64,
|
||||
objects: entry.objects as u64,
|
||||
versions: entry.versions as u64,
|
||||
delete_markers: entry.delete_markers as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add another set's counters into this one (entries are partitioned by
|
||||
/// set, so per-set results sum).
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
self.size = self.size.saturating_add(other.size);
|
||||
self.objects = self.objects.saturating_add(other.objects);
|
||||
self.versions = self.versions.saturating_add(other.versions);
|
||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||
}
|
||||
}
|
||||
|
||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct PrefixUsageEntry {
|
||||
pub prefix: String,
|
||||
pub usage: PrefixUsageSummary,
|
||||
}
|
||||
|
||||
/// Result of [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageQuery {
|
||||
pub usage: PrefixUsageSummary,
|
||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
||||
/// but no sub-prefix breakdown exists on disk.
|
||||
pub compacted: bool,
|
||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
||||
pub truncated: bool,
|
||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||
}
|
||||
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
@@ -1148,21 +997,6 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
||||
/// full counters instead of first-level sizes only).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or
|
||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
||||
/// which the `compacted` flag reports so callers can say why the
|
||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
||||
/// `max_entries` and was cut (largest first).
|
||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||
}
|
||||
|
||||
pub fn force_compact(&mut self, limit: usize) {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
@@ -2064,126 +1898,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
||||
/// distinct counters so aggregation is observable.
|
||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
||||
cache.replace(
|
||||
path,
|
||||
parent,
|
||||
DataUsageEntry {
|
||||
size,
|
||||
objects,
|
||||
versions,
|
||||
delete_markers,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
insert("bucket", "", 0, 0, 0, 0);
|
||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
||||
cache
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let root = cache
|
||||
.prefix_usage("bucket", "", 100)
|
||||
.expect("root query must find the bucket entry");
|
||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
||||
assert_eq!(root.usage.objects, 5);
|
||||
assert_eq!(root.usage.versions, 7);
|
||||
assert_eq!(root.usage.delete_markers, 2);
|
||||
assert!(!root.compacted);
|
||||
assert!(!root.truncated);
|
||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
||||
// each flattened to its own subtree total.
|
||||
let names: Vec<(&str, u64)> = root
|
||||
.sub_prefixes
|
||||
.iter()
|
||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
||||
.collect();
|
||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
||||
assert_eq!(b.usage.size, 240);
|
||||
assert_eq!(b.usage.versions, 5);
|
||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["c", "d"]);
|
||||
|
||||
// Prefix slashes are normalized away.
|
||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
||||
assert_eq!(slashed.usage.size, 240);
|
||||
|
||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_reports_and_respects_truncation() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_marks_compacted_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 999,
|
||||
objects: 9,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
||||
assert_eq!(compacted.usage.size, 999);
|
||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
||||
// whole query, not unbounded recursion.
|
||||
let mut cache = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b".to_string());
|
||||
}
|
||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
||||
// The unaffected sibling still answers.
|
||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
||||
|
||||
// A child key with no entry (dangling link) is rejected rather than
|
||||
// silently dropped: half a tree would under-report usage.
|
||||
let mut dangling = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b/ghost".to_string());
|
||||
}
|
||||
assert!(
|
||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
||||
"a dangling child link must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
@@ -32,7 +32,6 @@ use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -52,11 +51,6 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
const TEST_PORT_RANGE: u16 = 40_000;
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
@@ -73,64 +67,6 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
struct PortAllocatorGuard;
|
||||
|
||||
impl PortAllocatorGuard {
|
||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
loop {
|
||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
remove_stale_port_allocator_lock();
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PortAllocatorGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port() -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
||||
return;
|
||||
};
|
||||
let Ok(modified) = metadata.modified() else {
|
||||
return;
|
||||
};
|
||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
@@ -572,21 +508,10 @@ impl RustFSTestEnvironment {
|
||||
/// Find an available port for the test
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let mut next_port = read_next_test_port();
|
||||
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
drop(listener);
|
||||
return Ok(port);
|
||||
}
|
||||
}
|
||||
|
||||
Err("no available E2E test port found".into())
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let port = listener.local_addr()?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
/// Kill any existing RustFS processes
|
||||
|
||||
@@ -67,6 +67,9 @@ type MetricValues = Arc<Mutex<BTreeMap<String, MetricPointVersions>>>;
|
||||
|
||||
const KIB: usize = 1024;
|
||||
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
||||
/// Physical bytes the erasure layer pulled from disk, emitted per shard read by
|
||||
/// `crates/ecstore/src/erasure/coding/decode.rs`.
|
||||
const SHARD_READ_BYTES_COUNTER: &str = "rustfs_io_get_object_shard_read_observed_bytes_total";
|
||||
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
@@ -146,6 +149,7 @@ struct OtlpMetricCollector {
|
||||
decode_values: MetricValues,
|
||||
fallback_values: MetricValues,
|
||||
decode_error_values: MetricValues,
|
||||
shard_read_values: MetricValues,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
@@ -157,10 +161,12 @@ impl OtlpMetricCollector {
|
||||
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let shard_read_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||
let task_values = values.clone();
|
||||
let task_decode_values = decode_values.clone();
|
||||
let task_fallback_values = fallback_values.clone();
|
||||
let task_decode_error_values = decode_error_values.clone();
|
||||
let task_shard_read_values = shard_read_values.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
@@ -170,6 +176,7 @@ impl OtlpMetricCollector {
|
||||
let decode_values = task_decode_values.clone();
|
||||
let fallback_values = task_fallback_values.clone();
|
||||
let decode_error_values = task_decode_error_values.clone();
|
||||
let shard_read_values = task_shard_read_values.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(
|
||||
@@ -181,6 +188,7 @@ impl OtlpMetricCollector {
|
||||
decode_values.clone(),
|
||||
fallback_values.clone(),
|
||||
decode_error_values.clone(),
|
||||
shard_read_values.clone(),
|
||||
)
|
||||
}),
|
||||
)
|
||||
@@ -194,10 +202,48 @@ impl OtlpMetricCollector {
|
||||
decode_values,
|
||||
fallback_values,
|
||||
decode_error_values,
|
||||
shard_read_values,
|
||||
task,
|
||||
})
|
||||
}
|
||||
|
||||
/// Total physical bytes read from disk across every shard-read label set.
|
||||
async fn shard_read_bytes_total(&self) -> u64 {
|
||||
self.shard_read_values
|
||||
.lock()
|
||||
.await
|
||||
.values()
|
||||
.map(|versions| versions.values().map(|(_, value)| *value).sum::<u64>())
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Waits until the shard-read counter stops advancing so a measurement window
|
||||
/// is not polluted by exports still in flight.
|
||||
///
|
||||
/// Requires several consecutive equal samples spanning more than one export
|
||||
/// interval (`RUSTFS_OBS_METER_INTERVAL=1`): a single unchanged sample only
|
||||
/// proves the latest export has not landed yet, which silently reads as "no
|
||||
/// disk reads happened" and makes any upper-bound assertion vacuous.
|
||||
async fn wait_for_shard_read_bytes_to_settle(&self) -> TestResult<u64> {
|
||||
const REQUIRED_STABLE_SAMPLES: usize = 5;
|
||||
let mut last = self.shard_read_bytes_total().await;
|
||||
let mut stable = 0;
|
||||
for _ in 0..60 {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let current = self.shard_read_bytes_total().await;
|
||||
if current == last {
|
||||
stable += 1;
|
||||
if stable >= REQUIRED_STABLE_SAMPLES {
|
||||
return Ok(current);
|
||||
}
|
||||
} else {
|
||||
stable = 0;
|
||||
last = current;
|
||||
}
|
||||
}
|
||||
Err("timed out waiting for shard-read byte counter to settle".into())
|
||||
}
|
||||
|
||||
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
||||
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
||||
}
|
||||
@@ -321,6 +367,7 @@ async fn handle_metric_export(
|
||||
decode_values: MetricValues,
|
||||
fallback_values: MetricValues,
|
||||
decode_error_values: MetricValues,
|
||||
shard_read_values: MetricValues,
|
||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
if request.uri().path() != "/v1/metrics" {
|
||||
return Ok(response(StatusCode::NOT_FOUND));
|
||||
@@ -354,7 +401,9 @@ async fn handle_metric_export(
|
||||
let mut decode_values = decode_values.lock().await;
|
||||
let mut fallback_values = fallback_values.lock().await;
|
||||
let mut decode_error_values = decode_error_values.lock().await;
|
||||
let mut shard_read_values = shard_read_values.lock().await;
|
||||
record_reader_path_metrics(&export, &mut values);
|
||||
record_shard_read_bytes_metrics(&export, &mut shard_read_values);
|
||||
record_msgpack_decode_metrics(&export, &mut decode_values);
|
||||
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
||||
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
||||
@@ -375,6 +424,50 @@ fn reader_path_metric_key(path: &str, object_class: &str, size_bucket: &str) ->
|
||||
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
||||
}
|
||||
|
||||
/// Accumulates `SHARD_READ_BYTES_COUNTER` across all label sets. Only the total
|
||||
/// matters: it is the number of physical bytes the erasure layer actually pulled
|
||||
/// from disk, which is what separates a bounded per-part read from a decode of
|
||||
/// the whole object.
|
||||
fn record_shard_read_bytes_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||
for resource_metrics in &export.resource_metrics {
|
||||
for scope_metrics in &resource_metrics.scope_metrics {
|
||||
for metric in &scope_metrics.metrics {
|
||||
if metric.name != SHARD_READ_BYTES_COUNTER {
|
||||
continue;
|
||||
}
|
||||
let Some(metric::Data::Sum(sum)) = &metric.data else {
|
||||
continue;
|
||||
};
|
||||
for point in &sum.data_points {
|
||||
let Some(number_data_point::Value::AsInt(value)) = point.value.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let value = u64::try_from(*value).unwrap_or_default();
|
||||
// Keyed by labels, not by position: point order within an export
|
||||
// is not guaranteed stable, so an index key would alias distinct
|
||||
// series across batches.
|
||||
let key = format!(
|
||||
"{}\u{1f}{}\u{1f}{}",
|
||||
attribute_string(&point.attributes, "path").unwrap_or_default(),
|
||||
attribute_string(&point.attributes, "role").unwrap_or_default(),
|
||||
attribute_string(&point.attributes, "outcome").unwrap_or_default(),
|
||||
);
|
||||
values
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.entry(point.start_time_unix_nano)
|
||||
.and_modify(|current| {
|
||||
if point.time_unix_nano >= current.0 {
|
||||
*current = (point.time_unix_nano, value);
|
||||
}
|
||||
})
|
||||
.or_insert((point.time_unix_nano, value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||
for resource_metrics in &export.resource_metrics {
|
||||
for scope_metrics in &resource_metrics.scope_metrics {
|
||||
@@ -1864,6 +1957,86 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A tail range over a compressed multipart object must read only the physical
|
||||
/// data it needs, not decode the object from byte zero.
|
||||
///
|
||||
/// The byte-exactness tests around this one stay green even if the seek path
|
||||
/// regresses into decoding from the start of the object: the bytes returned are
|
||||
/// still correct, only the read amplification explodes. This asserts the cost
|
||||
/// side, using `SHARD_READ_BYTES_COUNTER` — already emitted per shard read by the
|
||||
/// erasure layer, so no production code is instrumented for the test.
|
||||
///
|
||||
/// `get_compressed_offsets` skips whole preceding parts by their stored size and
|
||||
/// then seeks inside the covering part via its compression index, so a bounded
|
||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
||||
/// object.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-tail-range";
|
||||
cluster.create_test_bucket(bucket).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
let key = "multipart/tail-range.txt";
|
||||
let (body, _second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||
|
||||
// Establish that the object really took the compressed read path; otherwise a
|
||||
// small delta below would only prove compression never happened.
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let baseline = collector.wait_for_shard_read_bytes_to_settle().await?;
|
||||
|
||||
let tail_len = 4 * KIB;
|
||||
let start = body.len() - tail_len;
|
||||
let end = body.len() - 1;
|
||||
let range = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.range(format!("bytes={start}-{end}"))
|
||||
.send()
|
||||
.await?;
|
||||
let tail = range.body.collect().await?.into_bytes();
|
||||
assert_eq!(tail.as_ref(), &body[start..], "tail range returned wrong bytes");
|
||||
|
||||
let after = collector.wait_for_shard_read_bytes_to_settle().await?;
|
||||
let read_bytes = after.saturating_sub(baseline);
|
||||
|
||||
// A zero delta means the window caught nothing — an unexported counter, or a
|
||||
// read served without touching the erasure layer — which would make the upper
|
||||
// bound vacuously true. Fail instead of passing blind.
|
||||
assert!(
|
||||
read_bytes > 0,
|
||||
"no shard reads observed for the tail range; the budget assertion below would be vacuous"
|
||||
);
|
||||
|
||||
// Part 1 alone is MPU_PART_1_SIZE, so a whole-object decode cannot come in
|
||||
// under it. Half the logical size leaves generous headroom for erasure padding
|
||||
// and unrelated background reads while still failing loudly on a full decode.
|
||||
let budget = (body.len() / 2) as u64;
|
||||
assert!(
|
||||
read_bytes < budget,
|
||||
"tail range read {read_bytes} physical bytes for a {tail_len}-byte range (budget {budget}, object {} bytes): \
|
||||
the read is not bounded to the covering part",
|
||||
body.len()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||
|
||||
@@ -30,6 +30,7 @@ use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -355,6 +356,7 @@ async fn run_post_object_policy_case(
|
||||
/// smuggles one extra field the policy never declared, and the upload must be
|
||||
/// rejected with 403 AccessDenied naming the offending field.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_fields_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -482,6 +484,7 @@ async fn test_anonymous_post_object_rejects_fields_missing_from_policy_condition
|
||||
/// sends a different one, and the upload must be rejected with 400
|
||||
/// InvalidPolicyDocument naming the field.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_exact_condition_policy_mismatches()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -686,6 +689,7 @@ async fn test_anonymous_post_object_rejects_exact_condition_policy_mismatches()
|
||||
/// one of them with a different value, and the upload must be rejected with
|
||||
/// 400 InvalidPolicyDocument naming the mismatched field.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_policy_mismatches() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -753,6 +757,7 @@ async fn test_anonymous_post_object_rejects_object_lock_policy_mismatches() -> R
|
||||
/// exact values, the form sends a different parameter value, and the upload
|
||||
/// must be rejected with 400 InvalidPolicyDocument naming the parameter.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sse_kms_policy_mismatches() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -834,6 +839,7 @@ async fn test_anonymous_post_object_rejects_sse_kms_policy_mismatches() -> Resul
|
||||
/// NotImplemented (SSE-KMS POST uploads are not implemented), not with a
|
||||
/// policy error.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sse_kms_params_outside_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -888,6 +894,7 @@ async fn test_anonymous_post_object_rejects_sse_kms_params_outside_policy_condit
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -961,6 +968,7 @@ async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box<
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_requires_auth() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -994,6 +1002,7 @@ async fn test_anonymous_post_object_requires_auth() -> Result<(), Box<dyn std::e
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_honors_success_action_status() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1057,6 +1066,7 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_honors_success_action_redirect() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1129,6 +1139,7 @@ async fn test_anonymous_post_object_honors_success_action_redirect() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_defaults_to_no_content() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1174,6 +1185,7 @@ async fn test_anonymous_post_object_defaults_to_no_content() -> Result<(), Box<d
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1220,6 +1232,7 @@ async fn test_anonymous_post_object_rejects_sse_kms() -> Result<(), Box<dyn std:
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1277,6 +1290,7 @@ async fn test_anonymous_post_object_accepts_sse_s3() -> Result<(), Box<dyn std::
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1349,6 +1363,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1422,6 +1437,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sse_s3_policy_mismatch() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1472,6 +1488,7 @@ async fn test_anonymous_post_object_rejects_sse_s3_policy_mismatch() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_sse_s3_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1535,6 +1552,7 @@ async fn test_anonymous_post_object_accepts_sse_s3_missing_from_policy_condition
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1588,6 +1606,7 @@ async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_storage_class_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1638,6 +1657,7 @@ async fn test_anonymous_post_object_rejects_storage_class_missing_from_policy_co
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_invalid_storage_class_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -1689,6 +1709,7 @@ async fn test_anonymous_post_object_rejects_invalid_storage_class_value() -> Res
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_checksum_algorithm_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1744,6 +1765,7 @@ async fn test_anonymous_post_object_rejects_checksum_algorithm_missing_from_poli
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_checksum_algorithm_policy_mismatch()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1800,6 +1822,7 @@ async fn test_anonymous_post_object_rejects_checksum_algorithm_policy_mismatch()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_checksum_auxiliary_fields_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1863,6 +1886,7 @@ async fn test_anonymous_post_object_rejects_checksum_auxiliary_fields_missing_fr
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -1939,6 +1963,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -1997,6 +2022,7 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_duplicate_key_form_values() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2046,6 +2072,7 @@ async fn test_anonymous_post_object_rejects_duplicate_key_form_values() -> Resul
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_invalid_success_action_status() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -2093,6 +2120,7 @@ async fn test_anonymous_post_object_rejects_invalid_success_action_status() -> R
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_invalid_success_action_redirect()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2140,6 +2168,7 @@ async fn test_anonymous_post_object_rejects_invalid_success_action_redirect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_form_fields_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2194,6 +2223,7 @@ async fn test_anonymous_post_object_rejects_form_fields_missing_from_policy_cond
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_form_fields_covered_by_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2250,6 +2280,7 @@ async fn test_anonymous_post_object_accepts_form_fields_covered_by_policy_condit
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_starts_with_policy_mismatch() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -2304,6 +2335,7 @@ async fn test_anonymous_post_object_rejects_starts_with_policy_mismatch() -> Res
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_length_range_violation()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2356,6 +2388,7 @@ async fn test_anonymous_post_object_rejects_content_length_range_violation()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_success_action_status_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2412,6 +2445,7 @@ async fn test_anonymous_post_object_accepts_success_action_status_exact_policy_m
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_success_action_redirect_policy_mismatch()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2468,6 +2502,7 @@ async fn test_anonymous_post_object_rejects_success_action_redirect_policy_misma
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_success_action_redirect_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2533,6 +2568,7 @@ async fn test_anonymous_post_object_accepts_success_action_redirect_exact_policy
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_success_action_redirect_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2585,6 +2621,7 @@ async fn test_anonymous_post_object_rejects_success_action_redirect_missing_from
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_metadata_field_covered_by_starts_with()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2639,6 +2676,7 @@ async fn test_anonymous_post_object_accepts_metadata_field_covered_by_starts_wit
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_type_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2696,6 +2734,7 @@ async fn test_anonymous_post_object_accepts_content_type_field_exact_policy_matc
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_type_field_covered_by_starts_with()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2753,6 +2792,7 @@ async fn test_anonymous_post_object_accepts_content_type_field_covered_by_starts
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_disposition_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2810,6 +2850,7 @@ async fn test_anonymous_post_object_accepts_content_disposition_field_exact_poli
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_cache_control_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2867,6 +2908,7 @@ async fn test_anonymous_post_object_accepts_cache_control_field_exact_policy_mat
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2924,6 +2966,7 @@ async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -2981,6 +3024,7 @@ async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_website_redirect_location_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3038,6 +3082,7 @@ async fn test_anonymous_post_object_accepts_website_redirect_location_exact_poli
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_expires_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3095,6 +3140,7 @@ async fn test_anonymous_post_object_accepts_expires_field_exact_policy_match()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_retention_without_permission()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3150,6 +3196,7 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_without_permis
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3209,6 +3256,7 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_p
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permission()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3261,6 +3309,7 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_legal_hold_policy_mismatch()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3319,6 +3368,7 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_policy_mismat
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_legal_hold_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3376,6 +3426,7 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_missing_from_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_tagging_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3441,6 +3492,7 @@ async fn test_anonymous_post_object_accepts_tagging_field_exact_policy_match()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_metadata_field_exact_policy_match()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3499,6 +3551,7 @@ async fn test_anonymous_post_object_accepts_metadata_field_exact_policy_match()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3551,6 +3604,7 @@ async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_condit
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_sigv4_date_policy_mismatch() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3603,6 +3657,7 @@ async fn test_anonymous_post_object_rejects_sigv4_date_policy_mismatch() -> Resu
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_mismatched_bucket_form_field() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -3657,6 +3712,7 @@ async fn test_anonymous_post_object_rejects_mismatched_bucket_form_field() -> Re
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_multiple_bucket_values() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3708,6 +3764,7 @@ async fn test_anonymous_post_object_rejects_multiple_bucket_values() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_extra_content_disposition_field()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3763,6 +3820,7 @@ async fn test_anonymous_post_object_rejects_extra_content_disposition_field()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3833,6 +3891,7 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3897,6 +3956,7 @@ async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3944,6 +4004,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3986,6 +4047,7 @@ async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(),
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_rejects_invalid_storage_class() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4021,6 +4083,7 @@ async fn test_signed_put_object_extract_rejects_invalid_storage_class() -> Resul
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_rejects_write_offset_bytes_header() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4074,6 +4137,7 @@ async fn test_signed_put_object_rejects_write_offset_bytes_header() -> Result<()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_raw_signed_put_object_write_offset_bytes_returns_minio_compatible_error_body()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4112,6 +4176,7 @@ async fn test_raw_signed_put_object_write_offset_bytes_returns_minio_compatible_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_put_object_write_offset_bytes_returns_minio_compatible_error_body()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4170,6 +4235,7 @@ async fn test_anonymous_put_object_write_offset_bytes_returns_minio_compatible_e
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4234,6 +4300,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_rejects_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4289,6 +4356,7 @@ async fn test_signed_put_object_extract_rejects_bucket_default_sse_kms() -> Resu
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4353,6 +4421,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_object_lock_legal_hold() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -4407,6 +4476,7 @@ async fn test_signed_put_object_extract_preserves_object_lock_legal_hold() -> Re
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -4466,6 +4536,7 @@ async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Res
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_pax_retention_overrides_request_retention()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4529,6 +4600,7 @@ async fn test_signed_put_object_extract_pax_retention_overrides_request_retentio
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4562,6 +4634,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4597,6 +4670,7 @@ async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Bo
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4650,6 +4724,7 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4959,6 +5034,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5000,6 +5076,7 @@ async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Bo
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_preserves_directory_markers_by_default()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -5060,6 +5137,7 @@ async fn test_signed_put_object_extract_preserves_directory_markers_by_default()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tar_gz_archive() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5111,6 +5189,7 @@ async fn test_signed_put_object_extract_expands_tar_gz_archive() -> Result<(), B
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tgz_archive() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5162,6 +5241,7 @@ async fn test_signed_put_object_extract_expands_tgz_archive() -> Result<(), Box<
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tbz2_archive() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5213,6 +5293,7 @@ async fn test_signed_put_object_extract_expands_tbz2_archive() -> Result<(), Box
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_txz_archive() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5264,6 +5345,7 @@ async fn test_signed_put_object_extract_expands_txz_archive() -> Result<(), Box<
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -5337,6 +5419,7 @@ async fn test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_e
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_normalizes_prefix_header_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5379,6 +5462,7 @@ async fn test_signed_put_object_extract_normalizes_prefix_header_value() -> Resu
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5430,6 +5514,7 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -5463,6 +5548,7 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ use aws_sdk_s3::types::{
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -106,6 +107,7 @@ fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc>
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by COMPLIANCE retention");
|
||||
@@ -143,6 +145,7 @@ async fn test_delete_object_blocked_by_compliance_retention() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_governance_without_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by GOVERNANCE retention without bypass");
|
||||
@@ -172,6 +175,7 @@ async fn test_delete_object_blocked_by_governance_without_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_allowed_by_governance_with_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject allowed by GOVERNANCE retention with bypass");
|
||||
@@ -211,6 +215,7 @@ async fn test_delete_object_allowed_by_governance_with_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_creates_delete_marker_for_retained_current_version() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject creates delete marker for retained current version");
|
||||
@@ -261,6 +266,7 @@ async fn test_delete_object_creates_delete_marker_for_retained_current_version()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_blocked_by_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject blocked by Legal Hold");
|
||||
@@ -293,6 +299,7 @@ async fn test_delete_object_blocked_by_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_allowed_with_legal_hold_off() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject allowed with Legal Hold OFF");
|
||||
@@ -328,6 +335,7 @@ async fn test_delete_object_allowed_with_legal_hold_off() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_after_legal_hold_removed() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject succeeds after Legal Hold is removed");
|
||||
@@ -361,6 +369,7 @@ async fn test_delete_object_after_legal_hold_removed() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_legal_hold_returns_updated_status() {
|
||||
init_logging();
|
||||
info!("🧪 Test: GetObjectLegalHold returns updated status");
|
||||
@@ -416,6 +425,7 @@ async fn test_get_object_legal_hold_returns_updated_status() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_retention_returns_configured_values() {
|
||||
init_logging();
|
||||
info!("🧪 Test: GetObjectRetention returns configured values");
|
||||
@@ -466,6 +476,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
// creating a new current version. The lock protects the existing version
|
||||
// from deletion; it never blocks new versions.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObject overwrite of a legal-hold version creates a new version");
|
||||
@@ -550,6 +561,7 @@ async fn test_put_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_applies_requested_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject applies requested Legal Hold");
|
||||
@@ -601,6 +613,7 @@ async fn test_copy_object_applies_requested_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_does_not_inherit_source_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject does not inherit source Legal Hold");
|
||||
@@ -694,6 +707,7 @@ async fn test_copy_object_does_not_inherit_source_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject overwrite of a legal-hold destination creates a new version");
|
||||
@@ -773,6 +787,7 @@ async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_create_multipart_upload_applies_requested_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CreateMultipartUpload applies requested Legal Hold");
|
||||
@@ -838,6 +853,7 @@ async fn test_create_multipart_upload_applies_requested_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_create_multipart_upload_creates_new_version_under_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CreateMultipartUpload over a COMPLIANCE-retained key creates a new version");
|
||||
@@ -917,6 +933,7 @@ async fn test_create_multipart_upload_creates_new_version_under_compliance_reten
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Delete completed multipart object blocked by Legal Hold");
|
||||
@@ -976,6 +993,7 @@ async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_completed_multipart_object_blocked_by_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Delete completed multipart object blocked by retention");
|
||||
@@ -1037,6 +1055,7 @@ async fn test_delete_completed_multipart_object_blocked_by_retention() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under Legal Hold");
|
||||
@@ -1116,6 +1135,7 @@ async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_complete_multipart_upload_creates_new_version_under_compliance_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under COMPLIANCE retention");
|
||||
@@ -1189,6 +1209,7 @@ async fn test_complete_multipart_upload_creates_new_version_under_compliance_ret
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_paths_require_put_object_legal_hold_permission() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths require PutObjectLegalHold permission");
|
||||
@@ -1252,6 +1273,7 @@ async fn test_write_paths_require_put_object_legal_hold_permission() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_paths_require_put_object_retention_permission() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths require PutObjectRetention permission");
|
||||
@@ -1323,6 +1345,7 @@ async fn test_write_paths_require_put_object_retention_permission() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_objects_mixed_locked_unlocked() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObjects with mixed locked and unlocked objects");
|
||||
@@ -1404,6 +1427,7 @@ async fn test_delete_objects_mixed_locked_unlocked() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_compliance_cannot_shorten() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention cannot shorten COMPLIANCE retention");
|
||||
@@ -1444,6 +1468,7 @@ async fn test_put_retention_compliance_cannot_shorten() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_compliance_can_extend() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention can extend COMPLIANCE retention");
|
||||
@@ -1484,6 +1509,7 @@ async fn test_put_retention_compliance_can_extend() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_governance_extend_without_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention on GOVERNANCE can extend without bypass");
|
||||
@@ -1527,6 +1553,7 @@ async fn test_put_retention_governance_extend_without_bypass() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_retention_governance_shorten_requires_bypass() {
|
||||
init_logging();
|
||||
info!("🧪 Test: PutObjectRetention on GOVERNANCE requires bypass to shorten");
|
||||
@@ -1588,6 +1615,7 @@ async fn test_put_retention_governance_shorten_requires_bypass() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_default_retention_applied_to_new_objects() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Default retention is applied to new objects");
|
||||
@@ -1657,6 +1685,7 @@ async fn test_default_retention_applied_to_new_objects() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_object_creates_delete_marker_for_default_retained_current_version() {
|
||||
init_logging();
|
||||
info!("🧪 Test: DeleteObject creates delete marker for default-retained current version");
|
||||
@@ -1741,6 +1770,7 @@ async fn test_delete_object_creates_delete_marker_for_default_retained_current_v
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() {
|
||||
init_logging();
|
||||
info!("🧪 Test: write paths reject incomplete Object Lock retention headers");
|
||||
@@ -1839,6 +1869,7 @@ async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_retention_uses_destination_policy() {
|
||||
init_logging();
|
||||
info!("🧪 Test: CopyObject retention follows destination policy");
|
||||
@@ -2020,6 +2051,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_multipart_default_retention_fixed_at_create() {
|
||||
init_logging();
|
||||
info!("🧪 Test: multipart default retention is fixed at CreateMultipartUpload");
|
||||
@@ -2090,6 +2122,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)");
|
||||
@@ -2210,6 +2243,7 @@ async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioning_auto_enabled_with_object_lock() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Versioning is auto-enabled when Object Lock is configured");
|
||||
@@ -2268,6 +2302,7 @@ async fn test_versioning_auto_enabled_with_object_lock() {
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_error_message_distinguishes_legal_hold_from_retention() {
|
||||
init_logging();
|
||||
info!("🧪 Test: Error messages distinguish Legal Hold from Retention");
|
||||
|
||||
@@ -60,6 +60,7 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use s3s::header::X_AMZ_REPLICATION_STATUS;
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
@@ -2505,6 +2506,7 @@ async fn build_replication_pair(
|
||||
/// metadata was inherited wholesale from the source, so the scanner heal pass
|
||||
/// skipped it too — no PENDING/FAILED marker meant nothing to re-drive).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_replicates_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2553,6 +2555,7 @@ async fn test_copy_object_replicates_to_target() -> TestResult {
|
||||
/// independent object; every member must replicate to the remote target like a
|
||||
/// regular PUT (MinIO PutObjectExtract parity).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_snowball_extract_replicates_members_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -2598,6 +2601,7 @@ async fn test_snowball_extract_replicates_members_to_target() -> TestResult {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2634,6 +2638,7 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_rejects_target_without_object_lock() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2687,6 +2692,7 @@ async fn test_replication_check_rejects_target_without_object_lock() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2725,6 +2731,7 @@ async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2748,6 +2755,7 @@ async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_rejects_missing_replication_config() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2771,6 +2779,7 @@ async fn test_replication_check_rejects_missing_replication_config() -> Result<(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2789,6 +2798,7 @@ async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box<dyn E
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_same_bucket_on_same_deployment() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2832,6 +2842,7 @@ async fn test_set_remote_target_rejects_same_bucket_on_same_deployment() -> Resu
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2865,6 +2876,7 @@ async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_update_requires_arn() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -2916,6 +2928,7 @@ async fn test_set_remote_target_update_requires_arn() -> Result<(), Box<dyn Erro
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_update_rejects_missing_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3004,6 +3017,7 @@ async fn fetch_single_target(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_partial_update_preserves_credentials() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3089,6 +3103,7 @@ async fn test_set_remote_target_partial_update_preserves_credentials() -> Result
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3130,6 +3145,7 @@ async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box<d
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -3214,6 +3230,7 @@ async fn test_set_remote_target_rejects_self_signed_https_target_without_skip_tl
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_verify() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -3325,6 +3342,7 @@ async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_ve
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -3409,6 +3427,7 @@ async fn test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_allows_private_ca_https_target_with_ca_cert_pem() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3499,6 +3518,7 @@ async fn test_set_remote_target_allows_private_ca_https_target_with_ca_cert_pem(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3518,6 +3538,7 @@ async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box<dyn E
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_remote_targets_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3536,6 +3557,7 @@ async fn test_list_remote_targets_rejects_invalid_bucket() -> Result<(), Box<dyn
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_remove_remote_target_rejects_missing_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3576,6 +3598,7 @@ async fn test_remove_remote_target_rejects_missing_target() -> Result<(), Box<dy
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_remove_remote_target_rejects_missing_arn() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3600,6 +3623,7 @@ async fn test_remove_remote_target_rejects_missing_arn() -> Result<(), Box<dyn E
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_remove_remote_target_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3623,6 +3647,7 @@ async fn test_remove_remote_target_rejects_invalid_bucket() -> Result<(), Box<dy
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_remove_remote_target_rejects_target_used_by_replication() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3662,6 +3687,7 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3711,6 +3737,7 @@ async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Bo
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_replicates_put_object_issue_2539() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -3752,6 +3779,7 @@ async fn test_bucket_replication_replicates_put_object_issue_2539() -> Result<()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_converges_delete_marker_and_version_purge() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -3850,6 +3878,7 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -3936,6 +3965,7 @@ async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() ->
|
||||
/// interoperability profile for a runner that provisions MinIO credentials
|
||||
/// and a reachable endpoint.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4263,6 +4293,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_single_bucket_multipart_replication_fans_out_to_multiple_targets() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -4431,6 +4462,7 @@ async fn test_repl17_failure_observation_helpers() -> TestResult {
|
||||
/// the replica is decryptable only with the original customer key. The
|
||||
/// backlog#1291 property still holds: never a silent plaintext replica.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_contract() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4508,6 +4540,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
|
||||
/// part — part boundaries and the encrypted-multipart marker survive so the
|
||||
/// replica decrypts each part with its part-derived nonce.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4624,6 +4657,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
||||
/// (independent KMS, so success proves target-owned envelopes), preserved
|
||||
/// source ETag, and a version that stays stable across scanner cycles.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_s3_contract() -> TestResult {
|
||||
init_logging();
|
||||
assert_managed_sse_replicates_and_reencrypts("sse-s3", false).await
|
||||
@@ -4633,6 +4667,7 @@ async fn test_bucket_replication_sse_s3_contract() -> TestResult {
|
||||
/// fail closed — replication FAILED, and no plaintext (or any) replica ever
|
||||
/// materializes on the target.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_s3_fails_closed_without_target_kms() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4676,6 +4711,7 @@ async fn test_bucket_replication_sse_s3_fails_closed_without_target_kms() -> Tes
|
||||
/// the ETag comparison sees the preserved source ETag on the replica and does
|
||||
/// not rewrite it, so the replica's version stays stable through the resync.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_s3_resync_converges() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4732,6 +4768,7 @@ async fn test_bucket_replication_sse_s3_resync_converges() -> TestResult {
|
||||
/// re-encrypts under its own default key. The independent-KMS pair proves the
|
||||
/// replica's envelope is target-owned.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_kms_contract() -> TestResult {
|
||||
init_logging();
|
||||
assert_managed_sse_replicates_and_reencrypts("sse-kms", true).await
|
||||
@@ -4742,6 +4779,7 @@ async fn test_bucket_replication_sse_kms_contract() -> TestResult {
|
||||
/// carries the full header set (SSE intent, content-type, user metadata) and
|
||||
/// the completed replica preserves the source's multipart ETag.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_s3_multipart_reencrypts() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4835,6 +4873,7 @@ async fn test_bucket_replication_sse_s3_multipart_reencrypts() -> TestResult {
|
||||
/// still-running source's data scanner (short cycle via [`FAST_SCANNER_ENV`])
|
||||
/// re-drives the failed objects once the target is reachable again.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_recovers_after_target_outage() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -4914,6 +4953,7 @@ async fn test_bucket_replication_recovers_after_target_outage() -> TestResult {
|
||||
/// must settle back to zero even though the historical failed counter remains
|
||||
/// non-zero.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_backlog_metrics_observe_outage_and_recovery() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -5047,6 +5087,7 @@ async fn test_bucket_replication_backlog_metrics_observe_outage_and_recovery() -
|
||||
/// must converge every persisted failure, including the replayed delete marker
|
||||
/// (whose replication decision is re-derived from the live config).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_replays_failed_entries_after_source_restart() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -5138,6 +5179,7 @@ async fn test_bucket_replication_replays_failed_entries_after_source_restart() -
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_replayed_delete_marker_preserves_source_mtime_without_source_restart() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -5207,6 +5249,7 @@ async fn test_bucket_replication_replayed_delete_marker_preserves_source_mtime_w
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5250,6 +5293,7 @@ async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> R
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5293,6 +5337,7 @@ async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Re
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_allows_self_signed_https_with_skip_tls_verify_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -5371,6 +5416,7 @@ async fn test_site_replication_allows_self_signed_https_with_skip_tls_verify_rea
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -5449,6 +5495,7 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let resync_process_env = [
|
||||
@@ -5668,6 +5715,7 @@ async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -5916,6 +5964,7 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -6035,6 +6084,7 @@ async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -6143,6 +6193,7 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_replicates_object_with_bucket_versioning_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -6233,6 +6284,7 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
|
||||
/// receiver was dropped with only a debug line, while `replicate status` still reported
|
||||
/// "1/1 Buckets in sync" because both configs were byte-identical.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_config_broadcast_keeps_reverse_direction_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -6371,6 +6423,7 @@ async fn wait_for_site_replication_rule(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_active_active_converges_without_loops_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -6688,6 +6741,7 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -6775,6 +6829,7 @@ async fn test_site_replication_replicates_policy_backed_user_access_real_dual_no
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_replicates_group_policy_backed_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
@@ -6865,6 +6920,7 @@ async fn test_site_replication_replicates_group_policy_backed_access_real_dual_n
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_service_account_policy_from_accountinfo_round_trips_real_single_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -6916,6 +6972,7 @@ async fn test_service_account_policy_from_accountinfo_round_trips_real_single_no
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_replicates_multiple_service_accounts_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -7016,6 +7073,7 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7156,6 +7214,7 @@ async fn wait_for_target_request_version_id(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_resync_restart_revisits_objects_before_out_of_order_checkpoint() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7274,6 +7333,7 @@ async fn test_bucket_resync_restart_revisits_objects_before_out_of_order_checkpo
|
||||
/// CreateMultipartUpload (the version is decided at initiate time) must both
|
||||
/// carry the source version as `?versionId=`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_put_and_create_multipart_carry_source_version_id_query() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7388,6 +7448,7 @@ async fn test_replication_put_and_create_multipart_carry_source_version_id_query
|
||||
/// flow to the onward bucket, proving B's outbound replication and scanner
|
||||
/// are live.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_never_cascades_inbound_replicas() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7468,6 +7529,7 @@ async fn test_scanner_never_cascades_inbound_replicas() -> TestResult {
|
||||
/// version ids and still mint its own there — the check must not report OK
|
||||
/// while multipart deletes and heals would silently miss.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_multipart_only_version_minting_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7540,6 +7602,7 @@ async fn test_replication_check_flags_multipart_only_version_minting_target() ->
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7702,6 +7765,7 @@ async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult {
|
||||
/// BucketRemoteTargetVersionMismatch — while still cleaning up the probe
|
||||
/// object via the version id the target actually assigned.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -7942,6 +8006,7 @@ async fn wait_for_target_marker_purged(
|
||||
/// the target forever. Contract under test: a failed purge attempt is retried
|
||||
/// within the watch window and converges once the fault clears.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_retries_after_transient_target_failure() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-retry-src";
|
||||
@@ -8001,6 +8066,7 @@ async fn test_delayed_delete_marker_purge_retries_after_transient_target_failure
|
||||
/// with an idempotent 204, which used to look like success and strand the
|
||||
/// real marker on the target forever.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_uses_target_assigned_version() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-mint-src";
|
||||
@@ -8033,6 +8099,7 @@ async fn test_delayed_delete_marker_purge_uses_target_assigned_version() -> Test
|
||||
/// replayed purge succeeds, the entry must be acknowledged instead of being
|
||||
/// retained as Missed forever.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_exhaustion_persists_to_mrf_and_replays_on_restart() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-mrf-src";
|
||||
@@ -8169,6 +8236,7 @@ async fn build_scanner_compensation_pair(
|
||||
/// nil-version objects entirely (`scanner_folder.rs` heal_replication), so it
|
||||
/// must NEVER be compensated.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "scanner-comp-src";
|
||||
@@ -8284,6 +8352,7 @@ async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestR
|
||||
/// written after the rule replicate normally (the setting only gates the
|
||||
/// existing-object resync path).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_never_compensates_when_existing_object_replication_disabled() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "scanner-disabled-src";
|
||||
|
||||
@@ -273,7 +273,6 @@ proptest = "1"
|
||||
rcgen.workspace = true
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
tonic-prost = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] }
|
||||
|
||||
@@ -373,14 +373,14 @@ pub mod error {
|
||||
|
||||
pub mod erasure {
|
||||
pub use crate::erasure::coding::{
|
||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||
calc_shard_size, calc_shard_size_legacy,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod event {
|
||||
pub use crate::event::name::EventName;
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||
}
|
||||
|
||||
pub mod global {
|
||||
@@ -483,7 +483,6 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
|
||||
@@ -1549,8 +1549,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1561,7 +1561,6 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1697,7 +1696,6 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -456,23 +456,16 @@ impl<'a> LifecycleExpiryTrace<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ExpiryStats {
|
||||
pub fn missed_tasks(&self) -> i64 {
|
||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_free_vers_tasks(&self) -> i64 {
|
||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -1783,7 +1776,7 @@ impl TransitionState {
|
||||
.await;
|
||||
}
|
||||
global_metrics().record_scanner_transition_failed(1);
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||
error!(
|
||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
|
||||
@@ -80,10 +80,7 @@ impl LastDayTierStats {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m;
|
||||
|
||||
@@ -177,10 +177,9 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
struct ObjSweeper {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
bucket: String,
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
@@ -192,9 +191,9 @@ struct ObjSweeper {
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ObjSweeper {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||
Ok(Self {
|
||||
object: object.into(),
|
||||
@@ -203,20 +202,17 @@ impl ObjSweeper {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||
self.versioned = versioned;
|
||||
self.suspended = suspended;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id.clone(),
|
||||
@@ -230,7 +226,6 @@ impl ObjSweeper {
|
||||
opts
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||
self.transition_tier = info.tier;
|
||||
self.transition_status = info.status;
|
||||
@@ -271,7 +266,6 @@ impl ObjSweeper {
|
||||
None
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
|
||||
@@ -312,7 +312,9 @@ mod tests {
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
#[allow(dead_code)]
|
||||
quota: Option<u64>,
|
||||
#[allow(dead_code)]
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GetRequest {
|
||||
pub buffer: Vec<u8>,
|
||||
pub offset: i64,
|
||||
@@ -106,12 +107,11 @@ pub struct GetRequest {
|
||||
pub setting_object_info: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct GetResponse {
|
||||
pub size: i64,
|
||||
//pub error: error,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub did_read: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub object_info: ObjectInfo,
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
@@ -28,6 +27,7 @@ use tracing::warn;
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedGetOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub is_replication_ready_for_delete_marker: bool,
|
||||
@@ -77,7 +77,7 @@ impl GetObjectOptions {
|
||||
}
|
||||
}
|
||||
if self.checksum {
|
||||
headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
|
||||
headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
@@ -360,6 +360,7 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
with_versions: bool,
|
||||
|
||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||
@@ -149,7 +149,6 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||
@@ -260,7 +259,6 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -55,6 +55,7 @@ pub struct RemoveBucketOptions {
|
||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub replication_status: ReplicationStatus,
|
||||
@@ -464,10 +465,10 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct RemoveObjectError {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object_name: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
version_id: String,
|
||||
err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
@@ -372,8 +372,8 @@ pub struct Checksum {
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl Checksum {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Checksum {
|
||||
@@ -385,7 +385,7 @@ impl Checksum {
|
||||
Checksum::default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||
let b = match base64_decode(s.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
@@ -412,7 +412,7 @@ impl Checksum {
|
||||
base64_encode(&self.r)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn raw(&self) -> Option<Vec<u8>> {
|
||||
if !self.is_set() {
|
||||
return None;
|
||||
|
||||
@@ -37,17 +37,16 @@ pub struct PutObjReader {
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(reader: HashReader) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
self.reader = enc_reader;
|
||||
|
||||
|
||||
@@ -54,10 +54,6 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::{
|
||||
http::headers::{
|
||||
AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1,
|
||||
AMZ_CHECKSUM_SHA256,
|
||||
},
|
||||
net::get_endpoint_url,
|
||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||
};
|
||||
@@ -1387,12 +1383,12 @@ pub(crate) fn to_object_info_for_provider(
|
||||
};
|
||||
|
||||
// Extract checksums
|
||||
let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32);
|
||||
let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C);
|
||||
let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1);
|
||||
let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256);
|
||||
let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME);
|
||||
let checksum_mode = get_header(AMZ_CHECKSUM_MODE);
|
||||
let checksum_crc32 = get_header("x-amz-checksum-crc32");
|
||||
let checksum_crc32c = get_header("x-amz-checksum-crc32c");
|
||||
let checksum_sha1 = get_header("x-amz-checksum-sha1");
|
||||
let checksum_sha256 = get_header("x-amz-checksum-sha256");
|
||||
let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
|
||||
let checksum_mode = get_header("x-amz-checksum-mode");
|
||||
|
||||
// Build and return the ObjectInfo struct
|
||||
Ok(ObjectInfo {
|
||||
|
||||
@@ -233,17 +233,11 @@ pub struct NsScannerCapabilityRequest {
|
||||
#[async_trait]
|
||||
pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
|
||||
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
|
||||
self.open_read(request).await
|
||||
}
|
||||
/// Opens an owned-chunk stream when this transport can retain receive-buffer
|
||||
/// ownership. `None` preserves the established `open_read` fallback.
|
||||
async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
self.open_read_chunks(request).await
|
||||
}
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
|
||||
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
|
||||
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
|
||||
@@ -275,15 +269,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
))
|
||||
}
|
||||
|
||||
async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result<FileReader> {
|
||||
let url = build_read_file_stream_url(&request);
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
Ok(Box::new(
|
||||
HttpReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout).await?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
let url = build_read_file_stream_url(&request);
|
||||
let mut headers = json_headers();
|
||||
@@ -293,16 +278,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
)))
|
||||
}
|
||||
|
||||
async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result<Option<ChunkReaderBox>> {
|
||||
let url = build_read_file_stream_url(&request);
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
Ok(Some(Box::new(
|
||||
HttpChunkReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout)
|
||||
.await?,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let nonce = server_epoch.map(|_| Uuid::new_v4());
|
||||
|
||||
@@ -86,25 +86,6 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Error for a peer that reported `success = false` without an `error_info` payload.
|
||||
///
|
||||
/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError`
|
||||
/// instead of `DiskError`. The message names the operation (and the bucket, where the
|
||||
/// operation has one) and nothing else, for two reasons:
|
||||
///
|
||||
/// - `finalize_result` classifies failures by message substring, so any text matching
|
||||
/// `message_has_network_needle` would take an answering peer offline and evict its
|
||||
/// connection over a plain application-level rejection.
|
||||
/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered
|
||||
/// message, so a per-peer detail such as the peer address would split one shared
|
||||
/// failure into single-count buckets and downgrade the dominant error.
|
||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||
match bucket {
|
||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
@@ -715,7 +696,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("local_storage_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.storage_info;
|
||||
|
||||
@@ -738,7 +719,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("server_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.server_properties;
|
||||
|
||||
@@ -761,7 +742,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_cpus", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.cpus;
|
||||
|
||||
@@ -784,7 +765,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_net_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.net_info;
|
||||
|
||||
@@ -807,7 +788,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_partitions", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.partitions;
|
||||
|
||||
@@ -830,7 +811,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_os_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.os_info;
|
||||
|
||||
@@ -851,7 +832,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_se_linux_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.sys_services;
|
||||
|
||||
@@ -876,7 +857,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_sys_config", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.sys_config;
|
||||
|
||||
@@ -901,7 +882,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_sys_errors", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.sys_errors;
|
||||
|
||||
@@ -926,7 +907,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_mem_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.mem_info;
|
||||
|
||||
@@ -958,7 +939,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_metrics", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.realtime_metrics;
|
||||
|
||||
@@ -983,7 +964,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_live_events", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(PeerLiveEventsBatch {
|
||||
@@ -1008,7 +989,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("get_proc_info", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
let data = response.proc_info;
|
||||
|
||||
@@ -1035,7 +1016,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("start_profiling", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1342,7 +1323,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1365,7 +1346,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket)));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1388,7 +1369,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("delete_policy", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1411,7 +1392,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_policy", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1436,7 +1417,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_policy_mapping", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1459,7 +1440,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("delete_user", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1482,7 +1463,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("delete_service_account", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1506,7 +1487,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_user", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1529,7 +1510,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_service_account", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1552,7 +1533,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_group", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1573,7 +1554,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("reload_site_replication_config", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1616,7 +1597,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("signal_service", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
||||
Ok(response)
|
||||
@@ -1686,7 +1667,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("reload_pool_meta", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1710,7 +1691,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("stop_rebalance", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1744,7 +1725,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_rebalance_meta", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1772,7 +1753,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("start_decommission", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1796,7 +1777,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("decommission_cancel", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1820,7 +1801,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("clear_decommission", None));
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1966,8 +1947,6 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error_reduce::reduce_errs;
|
||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||
use serde_json::Value;
|
||||
@@ -3119,115 +3098,4 @@ mod tests {
|
||||
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
||||
}));
|
||||
}
|
||||
|
||||
/// Every operation name passed to `peer_failure_without_details` in this file.
|
||||
const PEER_FAILURE_OPS: &[&str] = &[
|
||||
"local_storage_info",
|
||||
"server_info",
|
||||
"get_cpus",
|
||||
"get_net_info",
|
||||
"get_partitions",
|
||||
"get_os_info",
|
||||
"get_se_linux_info",
|
||||
"get_sys_config",
|
||||
"get_sys_errors",
|
||||
"get_mem_info",
|
||||
"get_metrics",
|
||||
"get_live_events",
|
||||
"get_proc_info",
|
||||
"start_profiling",
|
||||
"load_bucket_metadata",
|
||||
"delete_bucket_metadata",
|
||||
"delete_policy",
|
||||
"load_policy",
|
||||
"load_policy_mapping",
|
||||
"delete_user",
|
||||
"delete_service_account",
|
||||
"load_user",
|
||||
"load_service_account",
|
||||
"load_group",
|
||||
"reload_site_replication_config",
|
||||
"signal_service",
|
||||
"reload_pool_meta",
|
||||
"stop_rebalance",
|
||||
"load_rebalance_meta",
|
||||
"start_decommission",
|
||||
"decommission_cancel",
|
||||
"clear_decommission",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||
for op in PEER_FAILURE_OPS {
|
||||
let message = peer_failure_without_details(op, None).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
}
|
||||
|
||||
for op in ["load_bucket_metadata", "delete_bucket_metadata"] {
|
||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||
// same operation must stay a single dominant error instead of one bucket per peer.
|
||||
let per_peer_errs = (0..4)
|
||||
.map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))))
|
||||
.collect::<Vec<_>>();
|
||||
let (count, dominant) = reduce_errs(&per_peer_errs, &[]);
|
||||
assert_eq!(count, 4, "one shared failure must not split into per-peer buckets");
|
||||
assert_eq!(
|
||||
dominant,
|
||||
Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(),
|
||||
peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string()
|
||||
);
|
||||
assert_ne!(
|
||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(),
|
||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_never_reads_as_a_network_failure() {
|
||||
// `finalize_result` marks the peer offline and evicts its connection whenever the
|
||||
// message matches a network needle. A peer that answered `success = false` is alive,
|
||||
// so no operation or bucket name may push this text over that classifier.
|
||||
for op in PEER_FAILURE_OPS {
|
||||
let err = peer_failure_without_details(op, None);
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&err),
|
||||
"{op} must not read as a transport failure: {err}"
|
||||
);
|
||||
|
||||
let scoped = peer_failure_without_details(op, Some("bucket-name"));
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&scoped),
|
||||
"{op} must not read as a transport failure: {scoped}"
|
||||
);
|
||||
}
|
||||
|
||||
// The bucket name is caller-supplied. Every needle carries a space, which S3 bucket
|
||||
// names cannot, and the name is closed by `)` before the literal text resumes, so no
|
||||
// needle can straddle the boundary either.
|
||||
for bucket in [
|
||||
"timed-out",
|
||||
"connection-reset",
|
||||
"transport-error",
|
||||
"broken-pipe",
|
||||
"unavailable-logs",
|
||||
] {
|
||||
let err = peer_failure_without_details("load_bucket_metadata", Some(bucket));
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&err),
|
||||
"bucket {bucket} must not push the message over the network classifier: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,21 +214,6 @@ fn pool_write_quorum(participant_count: usize) -> usize {
|
||||
(participant_count / 2) + 1
|
||||
}
|
||||
|
||||
/// Error for a peer that reported `success = false` without an error payload.
|
||||
///
|
||||
/// The message must stay identical across the peers of one operation: `reduce_errs`
|
||||
/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address,
|
||||
/// timing) would split one shared failure into single-count buckets and downgrade a real
|
||||
/// dominant error into `ErasureWriteQuorum`.
|
||||
///
|
||||
/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape.
|
||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||
match bucket {
|
||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
||||
if per_pool_errs.is_empty() {
|
||||
return Some(Error::ErasureWriteQuorum);
|
||||
@@ -1093,7 +1078,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(peer_failure_without_details("heal_bucket", Some(bucket)))
|
||||
Err(Error::other(""))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1120,7 +1105,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(peer_failure_without_details("list_bucket", None))
|
||||
Err(Error::other(""))
|
||||
};
|
||||
}
|
||||
let bucket_infos = response
|
||||
@@ -1151,7 +1136,9 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(peer_failure_without_details("make_bucket", Some(bucket)))
|
||||
Err(Error::other(format!(
|
||||
"make_bucket({bucket}): peer returned failure without error details"
|
||||
)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1175,7 +1162,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(peer_failure_without_details("get_bucket_info", Some(bucket)))
|
||||
Err(Error::other(""))
|
||||
};
|
||||
}
|
||||
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
||||
@@ -1203,7 +1190,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(peer_failure_without_details("delete_bucket", Some(bucket)))
|
||||
Err(Error::other(""))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2327,37 +2314,4 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||
for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] {
|
||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||
}
|
||||
|
||||
let message = peer_failure_without_details("list_bucket", None).to_string();
|
||||
assert!(message.contains("list_bucket"), "cluster-wide message must name the operation");
|
||||
assert!(!message.trim().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||
// same operation on the same bucket must still reach quorum as one dominant error.
|
||||
let per_pool_errs = vec![
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
];
|
||||
assert_eq!(
|
||||
reduce_pool_write_quorum_errs(&per_pool_errs),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared")))
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
peer_failure_without_details("delete_bucket", Some("shared")),
|
||||
peer_failure_without_details("get_bucket_info", Some("shared"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,6 +39,7 @@ use rustfs_config::{
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit webhook settings.
|
||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -116,6 +117,7 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit MQTT settings.
|
||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -373,6 +375,7 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
|
||||
@@ -12,9 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![KV {
|
||||
@@ -23,3 +26,59 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
hidden_if_empty: false,
|
||||
}])
|
||||
});
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Config {
|
||||
pub bitrot: String,
|
||||
pub sleep: Duration,
|
||||
pub io_count: usize,
|
||||
pub drive_workers: usize,
|
||||
pub cache: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn bitrot_scan_cycle(&self) -> Duration {
|
||||
self.cache
|
||||
}
|
||||
|
||||
pub fn get_workers(&self) -> usize {
|
||||
self.drive_workers
|
||||
}
|
||||
|
||||
pub fn update(&mut self, nopts: &Config) {
|
||||
self.bitrot = nopts.bitrot.clone();
|
||||
self.io_count = nopts.io_count;
|
||||
self.sleep = nopts.sleep;
|
||||
self.drive_workers = nopts.drive_workers;
|
||||
}
|
||||
}
|
||||
|
||||
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
||||
|
||||
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
||||
match parse_bool(s) {
|
||||
Ok(enabled) => {
|
||||
if enabled {
|
||||
Ok(Duration::from_secs_f64(0.0))
|
||||
} else {
|
||||
Ok(Duration::from_secs_f64(-1.0))
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
if !s.ends_with("m") {
|
||||
return Err(Error::other("unknown format"));
|
||||
}
|
||||
|
||||
match s.trim_end_matches('m').parse::<u64>() {
|
||||
Ok(months) => {
|
||||
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
||||
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
||||
}
|
||||
|
||||
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
||||
}
|
||||
Err(err) => Err(Error::other(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
mod audit;
|
||||
pub mod com;
|
||||
#[allow(dead_code)]
|
||||
pub mod heal;
|
||||
mod notify;
|
||||
mod oidc;
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::bucket::replication::replication_state_from_filemeta;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::bucket::{
|
||||
lifecycle::{
|
||||
LifecycleExpiryConfigs,
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||
@@ -1997,11 +1996,11 @@ impl PoolMeta {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||
struct PoolInfo {
|
||||
position: usize,
|
||||
completed: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
decom_started: bool,
|
||||
}
|
||||
|
||||
@@ -2336,10 +2335,6 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
|
||||
action.delete()
|
||||
}
|
||||
|
||||
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
||||
if !apply_actions || applied {
|
||||
return Ok(true);
|
||||
@@ -2390,80 +2385,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
configs: LifecycleExpiryConfigs,
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
if bucket == RUSTFS_META_BUCKET {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let configs = get_expiry_configs(self, bucket).await?;
|
||||
if configs.lifecycle.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(HealLifecycleExpiryContext { configs }))
|
||||
}
|
||||
|
||||
pub async fn enqueue_heal_lifecycle_expiry(
|
||||
self: &Arc<Self>,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&crate::object_api::ObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let object_info = if let Some(object_info) = object_info {
|
||||
if object_info.bucket != bucket || object_info.name != object {
|
||||
return Ok(false);
|
||||
}
|
||||
let snapshot_version_id = object_info
|
||||
.version_id
|
||||
.filter(|version_id| !version_id.is_nil())
|
||||
.map(|version_id| version_id.to_string());
|
||||
if snapshot_version_id.as_deref() != version_id {
|
||||
return Ok(false);
|
||||
}
|
||||
object_info.clone()
|
||||
} else {
|
||||
match self
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.map(str::to_string),
|
||||
versioned: version_id.is_some(),
|
||||
expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(object_info) => object_info,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
};
|
||||
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
|
||||
if !lifecycle_action_skips_heal_version(event.action) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
|
||||
}
|
||||
|
||||
async fn save_current_pool_meta(&self) -> Result<()> {
|
||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let snapshot = {
|
||||
@@ -4365,19 +4287,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_action_skips_heal_version_for_every_delete_action() {
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
||||
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
||||
@@ -5049,19 +4958,13 @@ fn is_disk_online_state(state: &str) -> bool {
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_total_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_free_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
@@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
self.get_disks_for_heal_object(object, opts)?
|
||||
.check_abandoned_parts(bucket, object, opts)
|
||||
.await
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the pool/set layers
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1996,7 +1996,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sets_check_abandoned_parts_rejects_invalid_set_scope() {
|
||||
async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let sets = Sets {
|
||||
id: format.id,
|
||||
@@ -2021,21 +2021,10 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = sets
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
.await
|
||||
.expect_err("out-of-range abandoned-parts set scope must fail closed");
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "set" && reason.contains("invalid heal set index 1")),
|
||||
"unexpected invalid set error: {err:?}"
|
||||
);
|
||||
.expect_err("abandoned-parts ownership should stay above the pool/set storage layers");
|
||||
assert!(matches!(err, StorageError::NotImplemented));
|
||||
}
|
||||
|
||||
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
||||
|
||||
@@ -418,17 +418,6 @@ pub struct DiskHealthTracker {
|
||||
pub last_capacity_free: AtomicU64,
|
||||
/// Last successful capacity probe timestamp
|
||||
pub last_capacity_probe_unix_secs: AtomicI64,
|
||||
/// Authoritative atomically published runtime/status pair.
|
||||
state_snapshot: AtomicU64,
|
||||
transition_lock: std::sync::Mutex<()>,
|
||||
}
|
||||
|
||||
fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 {
|
||||
(u64::from(runtime_state as u32) << 32) | u64::from(status)
|
||||
}
|
||||
|
||||
fn unpack_health_state(snapshot: u64) -> (RuntimeDriveHealthState, u32) {
|
||||
(RuntimeDriveHealthState::from_u32((snapshot >> 32) as u32), snapshot as u32)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -750,8 +739,6 @@ impl DiskHealthTracker {
|
||||
last_capacity_used: AtomicU64::new(0),
|
||||
last_capacity_free: AtomicU64::new(0),
|
||||
last_capacity_probe_unix_secs: AtomicI64::new(0),
|
||||
state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)),
|
||||
transition_lock: std::sync::Mutex::new(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -788,52 +775,39 @@ impl DiskHealthTracker {
|
||||
|
||||
/// Check if disk is faulty
|
||||
pub fn is_faulty(&self) -> bool {
|
||||
unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).1 == DISK_HEALTH_FAULTY
|
||||
}
|
||||
|
||||
fn publish_state(&self, runtime_state: RuntimeDriveHealthState, status: u32) {
|
||||
self.state_snapshot
|
||||
.store(pack_health_state(runtime_state, status), Ordering::Release);
|
||||
self.runtime_state.store(runtime_state as u32, Ordering::Release);
|
||||
self.status.store(status, Ordering::Release);
|
||||
self.status.load(Ordering::Acquire) == DISK_HEALTH_FAULTY
|
||||
}
|
||||
|
||||
/// Set disk as faulty
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn set_faulty(&self) {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Set disk as OK
|
||||
pub fn set_ok(&self) {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
self.publish_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK);
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let status = if state == RuntimeDriveHealthState::Offline {
|
||||
DISK_HEALTH_FAULTY
|
||||
} else {
|
||||
DISK_HEALTH_OK
|
||||
};
|
||||
self.publish_state(state, status);
|
||||
self.runtime_state.store(state as u32, Ordering::Release);
|
||||
match state {
|
||||
RuntimeDriveHealthState::Offline => self.set_faulty(),
|
||||
RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
|
||||
self.set_ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn swap_ok_to_faulty(&self) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire));
|
||||
if status != DISK_HEALTH_OK {
|
||||
return false;
|
||||
}
|
||||
self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
|
||||
true
|
||||
self.status
|
||||
.compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
|
||||
unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).0
|
||||
RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
pub fn offline_duration(&self) -> Option<Duration> {
|
||||
@@ -849,7 +823,6 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
let now = current_unix_secs();
|
||||
let next = match current {
|
||||
@@ -878,19 +851,24 @@ impl DiskHealthTracker {
|
||||
};
|
||||
|
||||
let became_offline = next == RuntimeDriveHealthState::Offline && current != RuntimeDriveHealthState::Offline;
|
||||
if next == RuntimeDriveHealthState::Offline {
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
} else {
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
}
|
||||
self.transition_state(endpoint, current, next, reason);
|
||||
became_offline
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
if current == RuntimeDriveHealthState::Offline {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
self.transition_state(endpoint, current, RuntimeDriveHealthState::Offline, reason);
|
||||
true
|
||||
}
|
||||
@@ -904,10 +882,11 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let now_nanos = unix_nanos(now);
|
||||
let now_secs = unix_secs_i64(now);
|
||||
self.publish_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK);
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
self.runtime_state
|
||||
.store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
|
||||
self.consecutive_failures.store(0, Ordering::Release);
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
self.offline_since_unix_secs.store(0, Ordering::Release);
|
||||
@@ -919,7 +898,6 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
let next = match current {
|
||||
RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
|
||||
@@ -940,6 +918,7 @@ impl DiskHealthTracker {
|
||||
|
||||
let became_online = next == RuntimeDriveHealthState::Online;
|
||||
if became_online {
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
self.consecutive_failures.store(0, Ordering::Release);
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
}
|
||||
@@ -969,13 +948,7 @@ impl DiskHealthTracker {
|
||||
return;
|
||||
}
|
||||
|
||||
let current_status = unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).1;
|
||||
let status = match next {
|
||||
RuntimeDriveHealthState::Offline => DISK_HEALTH_FAULTY,
|
||||
RuntimeDriveHealthState::Returning => current_status,
|
||||
RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect => DISK_HEALTH_OK,
|
||||
};
|
||||
self.publish_state(next, status);
|
||||
self.runtime_state.store(next as u32, Ordering::Release);
|
||||
self.last_transition_unix_secs
|
||||
.store(current_unix_secs() as i64, Ordering::Release);
|
||||
|
||||
@@ -1244,7 +1217,7 @@ impl LocalDiskWrapper {
|
||||
return;
|
||||
}
|
||||
|
||||
if health.is_faulty() {
|
||||
if health.status.load(Ordering::Relaxed) != DISK_HEALTH_OK {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -2936,57 +2909,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn concurrent_failure_and_recovery_publish_one_health_snapshot() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
|
||||
let endpoint = Endpoint::try_from("/tmp/concurrent-health-snapshot").expect("endpoint should parse");
|
||||
let health = Arc::new(DiskHealthTracker::new());
|
||||
let transition_guard = health
|
||||
.transition_lock
|
||||
.lock()
|
||||
.expect("health transition lock should not be poisoned");
|
||||
let start = Arc::new(std::sync::Barrier::new(3));
|
||||
let (completed_tx, completed_rx) = std::sync::mpsc::channel();
|
||||
let workers = (0..2)
|
||||
.map(|_| {
|
||||
let health = Arc::clone(&health);
|
||||
let endpoint = endpoint.clone();
|
||||
let start = Arc::clone(&start);
|
||||
let completed_tx = completed_tx.clone();
|
||||
std::thread::spawn(move || {
|
||||
start.wait();
|
||||
health.mark_failure(&endpoint, "concurrent_test");
|
||||
completed_tx.send(()).expect("completion receiver should remain available");
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
start.wait();
|
||||
assert!(
|
||||
matches!(
|
||||
completed_rx.recv_timeout(Duration::from_millis(250)),
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||
),
|
||||
"concurrent transitions must wait for the serialization lock"
|
||||
);
|
||||
drop(transition_guard);
|
||||
completed_rx
|
||||
.recv_timeout(Duration::from_secs(1))
|
||||
.expect("first failure transition should complete after lock release");
|
||||
completed_rx
|
||||
.recv_timeout(Duration::from_secs(1))
|
||||
.expect("second failure transition should complete after lock release");
|
||||
for worker in workers {
|
||||
worker.join().expect("health transition worker should not panic");
|
||||
}
|
||||
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
|
||||
assert!(health.is_faulty());
|
||||
assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 2);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_success_recovers_suspect_drive_without_faulting() {
|
||||
let endpoint = Endpoint::try_from("/tmp/runtime-state-suspect-success").expect("endpoint should parse");
|
||||
|
||||
@@ -6562,7 +6562,7 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn get_metrics(&self) -> DiskMetrics {
|
||||
DiskMetrics::default()
|
||||
}
|
||||
|
||||
@@ -820,263 +820,10 @@ impl BitrotWriterWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
||||
//
|
||||
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
||||
// key-handling regression) fails silently: every shard reads back "corrupt",
|
||||
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
||||
// which copy is healthy. The self-test below pins the algorithms the moment a
|
||||
// process starts, so a drifted build announces itself instead of quietly
|
||||
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
||||
// analysis-2026-08-16.md §6 HS-11.
|
||||
|
||||
/// Length of the deterministic self-test payload.
|
||||
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
||||
|
||||
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
||||
/// (the production default). Pinned so any platform or build where the
|
||||
/// implementation drifts fails startup instead of miss-hashing shards.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
||||
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
||||
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
||||
];
|
||||
|
||||
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
||||
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
||||
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
||||
];
|
||||
|
||||
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
||||
/// Highway digests above this one is externally verifiable, so it guards the
|
||||
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
||||
/// self-computed constants.
|
||||
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
||||
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
||||
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
||||
];
|
||||
|
||||
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
||||
/// platform and every run hashes the same 4096 bytes.
|
||||
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
||||
for byte in payload.iter_mut() {
|
||||
state ^= state >> 12;
|
||||
state ^= state << 25;
|
||||
state ^= state >> 27;
|
||||
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
||||
}
|
||||
payload
|
||||
}
|
||||
|
||||
/// Why a bitrot self-test failed.
|
||||
#[derive(Debug)]
|
||||
pub enum BitrotSelfTestError {
|
||||
/// A known-answer digest mismatched the pinned constant.
|
||||
KnownAnswerMismatch {
|
||||
algorithm: &'static str,
|
||||
got: String,
|
||||
want: String,
|
||||
},
|
||||
/// A freshly encoded shard failed `bitrot_verify`.
|
||||
RoundtripVerify { algorithm: &'static str, detail: String },
|
||||
/// A verified roundtrip read back different bytes than were written.
|
||||
RoundtripReadback { algorithm: &'static str },
|
||||
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
||||
TamperNotRejected {
|
||||
algorithm: &'static str,
|
||||
tampered: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BitrotSelfTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
||||
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
||||
}
|
||||
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
||||
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
||||
Self::TamperNotRejected { algorithm, tampered } => {
|
||||
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BitrotSelfTestError {}
|
||||
|
||||
fn self_test_hex(bytes: &[u8]) -> String {
|
||||
rustfs_utils::hex(bytes)
|
||||
}
|
||||
|
||||
// (kept as a named one-liner so every KAT failure site reads the same; the
|
||||
// underlying formatter is the shared `rustfs_utils::hex`)
|
||||
|
||||
/// Compare a digest against its pinned constant. Split out so a test can drive
|
||||
/// it with a wrong constant and prove the mismatch path fires.
|
||||
fn bitrot_kat_check(
|
||||
algorithm: &'static str,
|
||||
algo: &HashAlgorithm,
|
||||
payload: &[u8],
|
||||
expected: &[u8; 32],
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let digest = algo.hash_encode(payload);
|
||||
let digest = digest.as_ref();
|
||||
if digest.len() != expected.len() || digest != expected.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm,
|
||||
got: self_test_hex(digest),
|
||||
want: self_test_hex(expected),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
||||
/// every block back through `BitrotReader` comparing bytes.
|
||||
async fn bitrot_roundtrip_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encode failed: {err}"),
|
||||
})?;
|
||||
}
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
if encoded.len() != on_disk {
|
||||
return Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
||||
});
|
||||
}
|
||||
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: err.to_string(),
|
||||
})?;
|
||||
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
||||
let mut offset = 0usize;
|
||||
while offset < payload.len() {
|
||||
let want = shard_size.min(payload.len() - offset);
|
||||
let mut buf = vec![0u8; want];
|
||||
let read = reader
|
||||
.read(&mut buf)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("read back failed at offset {offset}: {err}"),
|
||||
})?;
|
||||
if read != want || buf[..read] != payload[offset..offset + read] {
|
||||
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
||||
}
|
||||
offset += read;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flip one byte and require `bitrot_verify` to reject the result.
|
||||
async fn bitrot_tamper_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
tampered: &'static str,
|
||||
flip_at: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("self-test encode should not fail");
|
||||
}
|
||||
let mut corrupt = writer.into_inner().into_inner();
|
||||
let flip_index = flip_at % corrupt.len();
|
||||
corrupt[flip_index] ^= 0x80;
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
||||
// The flipped byte must be rejected as a hash mismatch specifically, not
|
||||
// by any incidental read error: an in-memory cursor cannot fail reads,
|
||||
// so accepting any other failure here would mask a verify path that
|
||||
// errors out before it ever compares hashes.
|
||||
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
||||
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
||||
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify every bitrot algorithm this crate can write or verify in production:
|
||||
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
||||
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
||||
/// and the leading hash, while all three hashed algorithms reproduce their
|
||||
/// pinned known-answer digests.
|
||||
///
|
||||
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
||||
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
||||
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
||||
let payload = bitrot_self_test_payload();
|
||||
|
||||
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
||||
// itself, before any self-pinned constants are consulted.
|
||||
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
||||
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm: "SHA256",
|
||||
got: self_test_hex(abc.as_ref()),
|
||||
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
||||
});
|
||||
}
|
||||
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256S",
|
||||
&HashAlgorithm::HighwayHash256S,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
||||
)?;
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256SLegacy",
|
||||
&HashAlgorithm::HighwayHash256SLegacy,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
||||
)?;
|
||||
|
||||
for (algorithm, algo) in [
|
||||
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
||||
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
||||
] {
|
||||
// Full blocks plus a partial tail, exactly like a real part stripe.
|
||||
let tail_len = 2 * 1024 + 333;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
||||
// One flipped byte in the final data block, one in the first leading
|
||||
// hash: both must fail verification.
|
||||
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
||||
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
||||
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
};
|
||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||
use bytes::Bytes;
|
||||
@@ -1343,32 +1090,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_payload_is_deterministic() {
|
||||
// Two independent builds of the payload must agree byte for byte, or
|
||||
// the pinned known-answer digests below would be meaningless.
|
||||
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
||||
let payload = bitrot_self_test_payload();
|
||||
let wrong = [0u8; 32];
|
||||
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
||||
.expect_err("a zeroed digest must never match");
|
||||
match err {
|
||||
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
||||
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_self_test_passes() {
|
||||
bitrot_self_test()
|
||||
.await
|
||||
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||
let mut counting = VectoredCountingWriter::default();
|
||||
@@ -1468,7 +1189,7 @@ mod tests {
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
std::io::Cursor::new(corrupt),
|
||||
Cursor::new(corrupt),
|
||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||
data.len(),
|
||||
algo,
|
||||
@@ -1561,7 +1282,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut out = [0u8; 5];
|
||||
let err = reader
|
||||
.read(&mut out)
|
||||
@@ -1686,7 +1407,7 @@ mod tests {
|
||||
(HashAlgorithm::HighwayHash256, true),
|
||||
] {
|
||||
let label = format!("{algo:?}");
|
||||
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
||||
let writer = Cursor::new(Vec::<u8>::new());
|
||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||
w.write(&[7u8; 16]).await.unwrap();
|
||||
let written = w.into_inner().into_inner();
|
||||
@@ -1771,7 +1492,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
w.write(payload).await.unwrap();
|
||||
w.into_inner().into_inner()
|
||||
}
|
||||
@@ -1879,7 +1600,7 @@ mod tests {
|
||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
w.write(chunk).await.unwrap();
|
||||
}
|
||||
@@ -1953,14 +1674,14 @@ mod tests {
|
||||
w.write(&data).await.expect("write shard");
|
||||
|
||||
let mut via_read = vec![0u8; SHARD];
|
||||
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
.read(&mut via_read)
|
||||
.await
|
||||
.expect("read");
|
||||
|
||||
// A buffer with only capacity — no initialized bytes at all.
|
||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut via_append, SHARD)
|
||||
.await
|
||||
.expect("read_appending");
|
||||
@@ -1985,7 +1706,7 @@ mod tests {
|
||||
encoded.truncate(encoded.len() - 1);
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a truncated shard must not succeed");
|
||||
@@ -2011,7 +1732,7 @@ mod tests {
|
||||
encoded[last] ^= 0xff;
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a corrupt shard must not verify");
|
||||
@@ -2123,7 +1844,7 @@ mod tests {
|
||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||
);
|
||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||
let mut streamed = std::io::Cursor::new(encoded.clone());
|
||||
let mut streamed = Cursor::new(encoded.clone());
|
||||
assert!(
|
||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||
"a non-Bytes source must stay on the streaming path"
|
||||
@@ -2151,7 +1872,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut via_stream, SHARD)
|
||||
.await
|
||||
.expect("streaming read");
|
||||
|
||||
@@ -1116,14 +1116,6 @@ mod tests {
|
||||
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
||||
}
|
||||
|
||||
// The lifecycle transition worker relies on this arm alone to suppress the
|
||||
// closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would
|
||||
// silently turn shutdown races back into `error!` log spam.
|
||||
#[test]
|
||||
fn is_network_or_host_down_covers_closed_network_connection() {
|
||||
assert!(is_network_or_host_down("transition failed: use of closed network connection", false));
|
||||
}
|
||||
|
||||
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
||||
// every set unreachable) must NOT be classified as "all not found",
|
||||
// otherwise ListObjects silently returns an empty listing and masks a full
|
||||
|
||||
@@ -132,6 +132,7 @@ impl RebalanceStopPropagationRecord {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DiskStat {
|
||||
pub total_space: u64,
|
||||
|
||||
@@ -16,16 +16,8 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{fmt::Display, io};
|
||||
use tracing::info;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier config wire version stamped by the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||
@@ -272,6 +264,7 @@ impl Clone for TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfig {
|
||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||
Self {
|
||||
@@ -291,7 +284,6 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||
@@ -311,7 +303,6 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||
@@ -331,7 +322,6 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||
@@ -351,7 +341,6 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||
@@ -468,7 +457,7 @@ impl TierWasabi {
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
access_key: &str,
|
||||
@@ -539,7 +528,7 @@ pub struct TierMinIO {
|
||||
}
|
||||
|
||||
impl TierMinIO {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
endpoint: &str,
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfigMgr {
|
||||
pub fn msg_size(&self) -> usize {
|
||||
100
|
||||
|
||||
@@ -4860,14 +4860,6 @@ impl SetDisks {
|
||||
/// is best-effort maintenance: individual delete failures are logged and
|
||||
/// skipped rather than propagated.
|
||||
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
|
||||
}
|
||||
|
||||
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||
@@ -4975,20 +4967,6 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
if dry_run {
|
||||
removed += 1;
|
||||
debug!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
event = "heal_abandoned_parts",
|
||||
component = "ecstore",
|
||||
subsystem = "heal",
|
||||
state = "dry_run_matched",
|
||||
result = "matched",
|
||||
bucket, object, data_dir = %dir,
|
||||
"Heal abandoned parts dry-run matched orphaned data directory"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match disk
|
||||
.delete(
|
||||
bucket,
|
||||
|
||||
@@ -6998,100 +6998,6 @@ mod tests {
|
||||
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
||||
}
|
||||
|
||||
async fn recv_abandoned_parts_trace(
|
||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
state: &str,
|
||||
) -> rustfs_common::trace_bus::TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("abandoned-parts trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if event.kind == rustfs_common::trace_bus::TraceKind::Heal
|
||||
&& event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts
|
||||
&& event.bucket.as_deref() == Some(bucket)
|
||||
&& event.object.as_deref() == Some(object)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected abandoned-parts trace state {state} for {bucket}/{object}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_abandoned_parts_dry_run_counts_without_deleting() {
|
||||
let mut trace = rustfs_common::trace_bus::subscribe_trace_events();
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
|
||||
let object_dir = dir.path().join("bucket").join("obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
fs::create_dir_all(object_dir.join(live.to_string()))
|
||||
.await
|
||||
.expect("live data dir should be created");
|
||||
fs::create_dir_all(object_dir.join(orphan.to_string()))
|
||||
.await
|
||||
.expect("orphan data dir should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dry-run abandoned-parts check should succeed");
|
||||
let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await;
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true"));
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved");
|
||||
assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir");
|
||||
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("abandoned-parts check should reclaim stale data dir");
|
||||
let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await;
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false"));
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(
|
||||
object_dir.join(live.to_string()).exists(),
|
||||
"referenced data dir must remain after reclaim"
|
||||
);
|
||||
assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
@@ -12327,18 +12233,11 @@ mod tests {
|
||||
.expect_err("unsupported copy_object_part should return a typed error");
|
||||
assert!(matches!(copy_part_err, StorageError::NotImplemented));
|
||||
|
||||
set_disks
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
let abandoned_err = set_disks
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
.await
|
||||
.expect("abandoned-parts check should be callable on empty disk sets");
|
||||
.expect_err("abandoned-parts check should stay in the upper reconciliation layer");
|
||||
assert!(matches!(abandoned_err, StorageError::NotImplemented));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,7 +16,6 @@ use super::super::*;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -2058,61 +2057,11 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let _write_lock_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let removed = if opts.dry_run {
|
||||
self.dry_run_reclaim_orphan_data_dirs(bucket, object).await?
|
||||
} else {
|
||||
self.reclaim_orphan_data_dirs(bucket, object).await?
|
||||
};
|
||||
let state = if opts.dry_run && removed > 0 {
|
||||
"dry_run_matched"
|
||||
} else if removed > 0 {
|
||||
"reclaimed"
|
||||
} else {
|
||||
"checked"
|
||||
};
|
||||
let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX);
|
||||
|
||||
trace_emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts)
|
||||
.with_bucket(bucket)
|
||||
.with_object(object)
|
||||
.with_duration(started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("dry_run", opts.dry_run)
|
||||
.with_attr("data_dirs", data_dirs)
|
||||
});
|
||||
|
||||
if removed > 0 {
|
||||
trace!(
|
||||
event = "heal_abandoned_parts",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" },
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
data_dirs = removed,
|
||||
"Heal abandoned parts checked object data directories"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the set layer
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3297,223 +3246,4 @@ mod heal_result_report_tests {
|
||||
assert!(result.detail.contains("part 1"));
|
||||
assert!(result.detail.contains("bitrot_failure=true"));
|
||||
}
|
||||
|
||||
// HS-12 (backlog#1874): a versioned DELETE racing an object heal must never
|
||||
// resurrect the deleted version. The heal has real reconstruction work (a
|
||||
// shard of the doomed version is removed), so both sides touch the same
|
||||
// (bucket, object, data_dir); whichever order the ns write lock serializes
|
||||
// them in, the committed delete must win.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_racing_version_delete_never_resurrects_the_deleted_version() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "heal-race-delete-no-resurrect";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned bucket should be created");
|
||||
|
||||
let mut first_reader = PutObjReader::from_vec(vec![0x11; 1024 * 1024]);
|
||||
let first_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut first_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("first version should be written");
|
||||
let first_version = first_info
|
||||
.version_id
|
||||
.expect("versioned put should return the first version id")
|
||||
.to_string();
|
||||
|
||||
let mut second_reader = PutObjReader::from_vec(vec![0x22; 1024 * 1024]);
|
||||
let second_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut second_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("second version should be written");
|
||||
let second_version = second_info
|
||||
.version_id
|
||||
.expect("versioned put should return the second version id")
|
||||
.to_string();
|
||||
|
||||
// Damage one shard of the doomed version so the racing heal performs an
|
||||
// actual reconstruction over its data dir instead of an early exit.
|
||||
let doomed_source = disks[0]
|
||||
.read_version("", bucket, object, &first_version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("doomed version metadata should be readable");
|
||||
let doomed_data_dir = doomed_source
|
||||
.data_dir
|
||||
.expect("non-inline version should have a data directory");
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(doomed_data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("shard damage should be injected before the race");
|
||||
|
||||
let delete_set = set.clone();
|
||||
let (delete_res, heal_res) = tokio::join!(
|
||||
async {
|
||||
delete_set
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(first_version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
},
|
||||
async {
|
||||
set.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
},
|
||||
);
|
||||
delete_res.expect("version delete must succeed under lock serialization");
|
||||
// The heal may legitimately report a transient failure when the version
|
||||
// it was rebuilding disappears mid-flight; only the end state matters.
|
||||
drop(heal_res);
|
||||
|
||||
let resurrected = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(first_version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&resurrected, Err(Error::FileVersionNotFound) | Err(Error::ObjectNotFound(..))),
|
||||
"a racing heal must not resurrect the deleted version: {resurrected:?}"
|
||||
);
|
||||
|
||||
let survivor = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(second_version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version must remain readable after the race");
|
||||
assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact");
|
||||
}
|
||||
|
||||
// HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on
|
||||
// the same object. The overwrite's post-commit tail deletes the replaced
|
||||
// data dir without the ns lock (object.rs commit tail), which is exactly
|
||||
// the intersection the audit flagged: the heal must tolerate the tail race
|
||||
// (retryable outcome) and every committed overwrite must survive — the
|
||||
// final current version is exactly the last payload written.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "heal-race-put-overwrite";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
const ROUNDS: usize = 8;
|
||||
const PAYLOAD_SIZE: usize = 256 * 1024;
|
||||
let mut last_etag = String::new();
|
||||
for round in 0..ROUNDS {
|
||||
// Give the heal something to rebuild on alternating rounds: remove a
|
||||
// shard of the current data dir right before the race.
|
||||
if round % 2 == 1 {
|
||||
let current = disks[2]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("current metadata should be readable");
|
||||
if let Some(data_dir) = current.data_dir {
|
||||
let shard = temp_dirs[3]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
if shard.exists() {
|
||||
tokio::fs::remove_file(&shard)
|
||||
.await
|
||||
.expect("shard damage should be injectable mid-race");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let payload = vec![round as u8; PAYLOAD_SIZE];
|
||||
let mut put_reader = PutObjReader::from_vec(payload);
|
||||
let put_opts = ObjectOptions::default();
|
||||
let heal_opts = HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
};
|
||||
let (put_res, heal_res) = tokio::join!(
|
||||
set.put_object(bucket, object, &mut put_reader, &put_opts),
|
||||
set.heal_object(bucket, object, "", &heal_opts),
|
||||
);
|
||||
let put_info = put_res.expect("overwrite must succeed under lock serialization");
|
||||
last_etag = put_info.etag.clone().unwrap_or_default();
|
||||
// Heal outcome is unconstrained (may hit the tail race and report a
|
||||
// retryable error); the invariant is checked on the end state.
|
||||
drop(heal_res);
|
||||
}
|
||||
|
||||
let final_info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("object must remain readable after the race loop");
|
||||
assert_eq!(
|
||||
final_info.size, PAYLOAD_SIZE as i64,
|
||||
"final current version must be the last committed overwrite"
|
||||
);
|
||||
assert_eq!(
|
||||
final_info.etag.unwrap_or_default(),
|
||||
last_etag,
|
||||
"the racing heal loop must never leave a stale or resurrected current version"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
//! per-version `SetDisks::heal_object`.
|
||||
|
||||
use super::super::*;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
@@ -40,16 +39,12 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// it must not gate healing logic — the delete-marker vs data path is chosen
|
||||
/// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is
|
||||
/// normalized (nil/absent UUID => `None`).
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealWalkVersion {
|
||||
/// object key
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<ObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -68,7 +63,6 @@ struct HealWalkCollector {
|
||||
bucket: String,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
objects: Mutex<Vec<HealWalkObject>>,
|
||||
decode_error: Mutex<Option<DiskError>>,
|
||||
version_total: AtomicUsize,
|
||||
@@ -122,25 +116,10 @@ impl HealWalkCollector {
|
||||
|
||||
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
// Normalize: nil/absent version id => None.
|
||||
version_id: version_uuid.map(|u| u.to_string()),
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -194,26 +173,11 @@ impl HealWalkCollector {
|
||||
}
|
||||
};
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let vid = version_uuid.map(|u| u.to_string());
|
||||
let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string());
|
||||
if seen.insert(vid.clone()) {
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
version_id: vid,
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -291,7 +255,6 @@ impl SetDisks {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
|
||||
|
||||
@@ -301,7 +264,6 @@ impl SetDisks {
|
||||
bucket: bucket.to_string(),
|
||||
batch_objects,
|
||||
version_budget: version_budget.max(1),
|
||||
include_lifecycle_object_info,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -385,7 +347,6 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 2,
|
||||
version_budget: 2,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -427,8 +388,6 @@ mod tests {
|
||||
HealWalkVersion {
|
||||
name: name.to_string(),
|
||||
version_id: Some(id.to_string()),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: dm,
|
||||
}
|
||||
}
|
||||
@@ -532,7 +491,6 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 1000,
|
||||
version_budget: 10_000,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
version_total: AtomicUsize::new(0),
|
||||
decode_error: Mutex::new(None),
|
||||
@@ -609,7 +567,7 @@ mod tests {
|
||||
.expect("corrupt test metadata should be written");
|
||||
|
||||
let error = set_disks
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2, false)
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2)
|
||||
.await
|
||||
.expect_err("semantic metadata corruption must fail the heal disk walk");
|
||||
|
||||
|
||||
@@ -5845,14 +5845,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||
// MRF journal intent: partial-write recovery must survive a restart
|
||||
// (HS-01); the heal request below remains the in-memory fast path.
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
bucket,
|
||||
object,
|
||||
uuid::Uuid::try_parse(version_id).ok(),
|
||||
);
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
|
||||
@@ -1077,15 +1077,6 @@ impl SetDisks {
|
||||
"Recoverable decode error triggered read repair"
|
||||
);
|
||||
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
||||
// MRF journal intent: keeps a durable Urgent ECDecode
|
||||
// request alive across restarts even when the in-memory
|
||||
// read-repair request is dropped or lost (HS-01).
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
bucket,
|
||||
object,
|
||||
fi.version_id,
|
||||
);
|
||||
submit_read_repair_heal(
|
||||
bucket,
|
||||
object,
|
||||
|
||||
@@ -18,7 +18,6 @@ use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
|
||||
@@ -257,40 +256,13 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let object = encode_dir_object(object);
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
futures.push(pool.check_abandoned_parts(bucket, &object, opts));
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
for result in join_all(futures).await {
|
||||
if let Err(err) = result
|
||||
&& first_error.is_none()
|
||||
{
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = first_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
trace!(
|
||||
event = EVENT_HEAL_ABANDONED_PARTS,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = "completed",
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
"Heal abandoned parts completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
let _ = (bucket, object, opts);
|
||||
// Stale multipart reconciliation is already owned by the lifecycle-driven
|
||||
// background cleanup path in `bucket_lifecycle_ops.rs`. There is currently
|
||||
// no stable object-heal contract that should fan this request out through
|
||||
// pool/set storage layers, so keep the placeholder explicit at the ECStore
|
||||
// boundary instead of dispatching into lower layers.
|
||||
Err(StorageError::NotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ impl ECStore {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() {
|
||||
return Err(Error::other(format!(
|
||||
@@ -44,7 +43,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
self.pools[pool_idx].disk_set[set_idx]
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info)
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget)
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
@@ -216,16 +216,6 @@ impl std::fmt::Debug for ECStore {
|
||||
/// These delegate to the process-global statics. No local state — the globals
|
||||
/// remain the single source of truth until the migration is complete.
|
||||
impl ECStore {
|
||||
/// Every erasure set across all pools, pool-major order.
|
||||
///
|
||||
/// Read-only queries that must consult each set's own copy of a
|
||||
/// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate
|
||||
/// this instead of the hash-routed store path, which would always land
|
||||
/// on one set (rustfs/backlog#1872).
|
||||
pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
|
||||
self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
|
||||
}
|
||||
|
||||
/// Get server configuration (delegates to global)
|
||||
pub fn get_server_config(&self) -> Option<Config> {
|
||||
runtime_sources::server_config()
|
||||
|
||||
@@ -89,8 +89,6 @@ async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
+17
-100
@@ -66,37 +66,21 @@ struct HealTaskStatusPayload<'a> {
|
||||
summary: &'a str,
|
||||
items: &'a [HealResultItem],
|
||||
truncated: bool,
|
||||
/// Cursor for incremental consumption (HS-06): sequence of the next item
|
||||
/// to be produced. Absent on responses without sequencing (0).
|
||||
#[serde(skip_serializing_if = "u64_is_zero")]
|
||||
next_seq: u64,
|
||||
/// Oldest sequence still retained; with `truncated`, tells a lagging
|
||||
/// client where to restart its cursor.
|
||||
#[serde(skip_serializing_if = "u64_is_zero")]
|
||||
min_seq: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
progress: Option<&'a HealProgress>,
|
||||
}
|
||||
|
||||
fn u64_is_zero(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
fn encode_heal_task_status_payload(
|
||||
summary: &str,
|
||||
mut items: Vec<HealResultItem>,
|
||||
progress: Option<&HealProgress>,
|
||||
mut truncated: bool,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
) -> Result<(Vec<u8>, bool)> {
|
||||
loop {
|
||||
let data = serde_json::to_vec(&HealTaskStatusPayload {
|
||||
summary,
|
||||
items: &items,
|
||||
truncated,
|
||||
next_seq,
|
||||
min_seq,
|
||||
progress,
|
||||
})
|
||||
.map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
|
||||
@@ -125,10 +109,8 @@ fn encode_heal_status_response(
|
||||
progress: Option<&HealProgress>,
|
||||
detail: Option<String>,
|
||||
truncated: bool,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
) -> Result<(Vec<u8>, Option<String>)> {
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?;
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated)?;
|
||||
Ok((data, heal_status_detail(detail, truncated)))
|
||||
}
|
||||
|
||||
@@ -156,19 +138,8 @@ impl HealChannelProcessor {
|
||||
|
||||
/// Execute a token query directly against the manager.
|
||||
pub async fn execute_query_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
|
||||
self.execute_query_request_since(heal_path, client_token, None).await
|
||||
}
|
||||
|
||||
/// Incremental variant of [`Self::execute_query_request`] (HS-06).
|
||||
pub async fn execute_query_request_since(
|
||||
&self,
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<HealChannelResponse> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
self.process_query_request(heal_path, client_token, since_seq, response_tx)
|
||||
.await?;
|
||||
self.process_query_request(heal_path, client_token, response_tx).await?;
|
||||
response_rx
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("heal query channel closed: {err}")))?
|
||||
@@ -291,12 +262,8 @@ impl HealChannelProcessor {
|
||||
HealChannelCommand::Query {
|
||||
heal_path,
|
||||
client_token,
|
||||
since_seq,
|
||||
response_tx,
|
||||
} => {
|
||||
self.process_query_request(heal_path, client_token, since_seq, response_tx)
|
||||
.await
|
||||
}
|
||||
} => self.process_query_request(heal_path, client_token, response_tx).await,
|
||||
HealChannelCommand::Cancel {
|
||||
heal_path,
|
||||
client_token,
|
||||
@@ -417,7 +384,6 @@ impl HealChannelProcessor {
|
||||
&self,
|
||||
heal_path: String,
|
||||
client_token: String,
|
||||
since_seq: Option<u64>,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
@@ -432,118 +398,72 @@ impl HealChannelProcessor {
|
||||
);
|
||||
|
||||
let report = if heal_path.trim_matches('/').is_empty() {
|
||||
self.heal_manager.get_task_report_since(&client_token, since_seq).await
|
||||
self.heal_manager.get_task_report(&client_token).await
|
||||
} else {
|
||||
self.heal_manager
|
||||
.get_task_report_for_path_since(&heal_path, &client_token, since_seq)
|
||||
.await
|
||||
self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await
|
||||
};
|
||||
|
||||
let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report {
|
||||
let (summary, detail, items, truncated, progress) = match report {
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending | HealTaskStatus::Running,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
None,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
}) => ("running".to_string(), None, result_items, result_items_truncated, progress),
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Retrying { error, retry_attempt },
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Completed,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"finished".to_string(),
|
||||
None,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
}) => ("finished".to_string(), None, result_items, result_items_truncated, progress),
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Cancelled,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task cancelled".to_string()),
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Timeout,
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task timed out".to_string()),
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Failed { error },
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some(error),
|
||||
result_items,
|
||||
result_items_truncated,
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
),
|
||||
}) => ("stopped".to_string(), Some(error), result_items, result_items_truncated, progress),
|
||||
Err(crate::Error::TaskNotFound { .. }) => (
|
||||
"notFound".to_string(),
|
||||
Some("heal task not found or expired".to_string()),
|
||||
Vec::new(),
|
||||
false,
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
),
|
||||
Err(crate::Error::InvalidClientToken) => {
|
||||
let response = HealChannelResponse {
|
||||
@@ -570,8 +490,7 @@ impl HealChannelProcessor {
|
||||
}
|
||||
};
|
||||
|
||||
let (data, detail) =
|
||||
encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?;
|
||||
let (data, detail) = encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated)?;
|
||||
|
||||
let response = HealChannelResponse {
|
||||
request_id: client_token,
|
||||
@@ -693,8 +612,7 @@ impl HealChannelProcessor {
|
||||
HealRequestSource::Admin
|
||||
| HealRequestSource::AutoHeal
|
||||
| HealRequestSource::Internal
|
||||
| HealRequestSource::ReadRepair
|
||||
| HealRequestSource::Mrf => true,
|
||||
| HealRequestSource::ReadRepair => true,
|
||||
});
|
||||
|
||||
// Build HealOptions with all available fields
|
||||
@@ -849,7 +767,6 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> crate::Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
@@ -886,7 +803,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap();
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false).unwrap();
|
||||
|
||||
assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
|
||||
let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
|
||||
@@ -1656,7 +1573,7 @@ mod tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request("bucket".to_string(), "completed-token".to_string(), None, tx)
|
||||
.process_query_request("bucket".to_string(), "completed-token".to_string(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
@@ -1691,7 +1608,7 @@ mod tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request("bucket".to_string(), task_id.clone(), None, tx)
|
||||
.process_query_request("bucket".to_string(), task_id.clone(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
@@ -1724,7 +1641,7 @@ mod tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request("bucket".to_string(), "wrong-token".to_string(), None, tx)
|
||||
.process_query_request("bucket".to_string(), "wrong-token".to_string(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
@@ -1749,7 +1666,7 @@ mod tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request(String::new(), "wrong-token".to_string(), None, tx)
|
||||
.process_query_request(String::new(), "wrong-token".to_string(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
@@ -1786,7 +1703,7 @@ mod tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request(String::new(), task_id.clone(), None, tx)
|
||||
.process_query_request(String::new(), task_id.clone(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
|
||||
@@ -23,14 +23,13 @@ use crate::heal::{
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use metrics::{counter, gauge};
|
||||
use metrics::gauge;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
@@ -48,21 +47,6 @@ enum HealObjectOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn result_object_size_u64(result: &HealResultItem) -> u64 {
|
||||
u64::try_from(result.object_size).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60;
|
||||
const NANOS_PER_SECOND: i128 = 1_000_000_000;
|
||||
|
||||
fn should_skip_new_version(mod_time_unix_nanos: Option<i128>, started_at_secs: u64) -> bool {
|
||||
let Some(mod_time_unix_nanos) = mod_time_unix_nanos else {
|
||||
return false;
|
||||
};
|
||||
let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS);
|
||||
mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND)
|
||||
}
|
||||
|
||||
struct PageConcurrencyGuard {
|
||||
in_flight: Arc<AtomicUsize>,
|
||||
set_label: String,
|
||||
@@ -508,7 +492,6 @@ impl ErasureSetHealer {
|
||||
&mut skipped_objects,
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -675,7 +658,6 @@ impl ErasureSetHealer {
|
||||
skipped_objects: &mut u64,
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -728,7 +710,6 @@ impl ErasureSetHealer {
|
||||
// The end-of-pass summary reports the full failed/skipped counts.
|
||||
let mut transient_skip_samples_logged = 0_u64;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||
|
||||
// backlog#920: select the per-erasure-set DISK-WALK union enumerator when
|
||||
// the scan is Deep OR the request came from AutoHeal — these are the paths
|
||||
@@ -737,25 +718,17 @@ impl ErasureSetHealer {
|
||||
// which stays the default.
|
||||
let use_disk_walk =
|
||||
matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal);
|
||||
let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?;
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
// Get one page of object versions
|
||||
let (objects, next_token, is_truncated) = if use_disk_walk {
|
||||
self.storage
|
||||
.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
"",
|
||||
continuation_token.as_deref(),
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref())
|
||||
.await?
|
||||
} else {
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info)
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
|
||||
.await?
|
||||
};
|
||||
let page_is_empty = objects.is_empty();
|
||||
@@ -763,7 +736,6 @@ impl ErasureSetHealer {
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
// Capture the last version identity of this page for the anti-loop guard.
|
||||
let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone()));
|
||||
@@ -779,75 +751,6 @@ impl ErasureSetHealer {
|
||||
continue;
|
||||
}
|
||||
|
||||
if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_new_versions_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_new_version();
|
||||
progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_new_version",
|
||||
"Erasure set object version skipped because it was written after heal started"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(context) = lifecycle_expiry_context.as_ref()
|
||||
&& self
|
||||
.storage
|
||||
.enqueue_heal_lifecycle_expiry(
|
||||
context,
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
item.lifecycle_object_info.as_ref(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_ilm_expired();
|
||||
progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_ilm_expired",
|
||||
"Erasure set object version skipped because lifecycle expiry was queued"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
resume_manager
|
||||
.set_current_item(Some(bucket.to_string()), Some(item.name.clone()))
|
||||
.await?;
|
||||
@@ -874,7 +777,7 @@ impl ErasureSetHealer {
|
||||
|
||||
let _permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))),
|
||||
Err(err) => return (dedup_key, object_name, version_id, Err(err)),
|
||||
};
|
||||
|
||||
let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label);
|
||||
@@ -885,7 +788,7 @@ impl ErasureSetHealer {
|
||||
// recorded as skipped-ok rather than failed. The delete-marker
|
||||
// vs data path is chosen internally in ops/heal.rs.
|
||||
let result = if cancel_token.is_cancelled() {
|
||||
(0, Err(Error::TaskCancelled))
|
||||
Err(Error::TaskCancelled)
|
||||
} else {
|
||||
match storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
@@ -894,9 +797,8 @@ impl ErasureSetHealer {
|
||||
Ok((result, None))
|
||||
if target_outcomes_complete(&result, &target_endpoints) =>
|
||||
{
|
||||
let object_size = result_object_size_u64(&result);
|
||||
if !replacement_commit_evidence_required {
|
||||
(object_size, Ok(true))
|
||||
Ok(true)
|
||||
} else {
|
||||
match storage
|
||||
.replacement_targets_have_version(
|
||||
@@ -908,42 +810,27 @@ impl ErasureSetHealer {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => (object_size, Ok(true)),
|
||||
Ok(false) => (object_size, Err(Error::transient_skip(format!(
|
||||
Ok(true) => Ok(true),
|
||||
Ok(false) => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version"
|
||||
)))),
|
||||
Err(err) => (object_size, Err(Error::transient_skip(format!(
|
||||
))),
|
||||
Err(err) => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}"
|
||||
)))),
|
||||
))),
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok((result, None)) if !target_endpoints.is_empty() => (
|
||||
result_object_size_u64(&result),
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
),
|
||||
Ok((result, None)) => (result_object_size_u64(&result), Ok(true)),
|
||||
Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => {
|
||||
(result_object_size_u64(&result), Ok(false))
|
||||
}
|
||||
Ok((result, Some(err))) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (object_size, Ok(false)),
|
||||
HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||
}
|
||||
}
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (0, Ok(false)),
|
||||
HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!(
|
||||
Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
Ok((_result, None)) => Ok(true),
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false),
|
||||
Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => Ok(false),
|
||||
HealObjectOutcome::Transient => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
))),
|
||||
HealObjectOutcome::Failed => Err(err),
|
||||
},
|
||||
}
|
||||
};
|
||||
@@ -952,12 +839,11 @@ impl ErasureSetHealer {
|
||||
});
|
||||
}
|
||||
|
||||
let mut completed_in_page = 0usize;
|
||||
while let Some((key, object, version_id, result)) = page_tasks.next().await {
|
||||
let (object_size, result) = result;
|
||||
match result {
|
||||
Ok(true) => {
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -975,7 +861,6 @@ impl ErasureSetHealer {
|
||||
Ok(false) => {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -992,7 +877,6 @@ impl ErasureSetHealer {
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
*skipped_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_skipped_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -1009,7 +893,6 @@ impl ErasureSetHealer {
|
||||
}
|
||||
Err(err) => {
|
||||
*failed_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_failed_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -1028,11 +911,6 @@ impl ErasureSetHealer {
|
||||
|
||||
*processed_objects += 1;
|
||||
completed_in_page += 1;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
@@ -1086,9 +964,7 @@ impl ErasureSetHealer {
|
||||
progress.objects_scanned = state.total_objects;
|
||||
progress.objects_healed = state.successful_objects;
|
||||
progress.objects_failed = state.failed_objects;
|
||||
progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
|
||||
progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
|
||||
progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
|
||||
progress.bytes_processed = 0; // set to 0 for now, can be extended later
|
||||
progress.set_current_object(state.current_object.clone());
|
||||
}
|
||||
}
|
||||
@@ -1259,15 +1135,13 @@ mod resume_loop_tests {
|
||||
//! that emits programmable multi-version pages. These exercise the real loop
|
||||
//! logic (cursor seeding, per-version dedup, anti-loop guard, absence
|
||||
//! handling) — not merely a mock's own output.
|
||||
use super::{
|
||||
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
|
||||
};
|
||||
use super::{ErasureSetHealer, target_outcomes_complete};
|
||||
use crate::heal::progress::HealProgress;
|
||||
use crate::heal::resume::{
|
||||
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
|
||||
compose_key,
|
||||
};
|
||||
use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
@@ -1275,7 +1149,7 @@ mod resume_loop_tests {
|
||||
use crate::{Error, Result};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tempfile::TempDir;
|
||||
@@ -1286,37 +1160,10 @@ mod resume_loop_tests {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: delete_marker,
|
||||
}
|
||||
}
|
||||
|
||||
fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)),
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_version_filter_respects_grace_boundary() {
|
||||
let started_at = 1_700_000_000;
|
||||
|
||||
assert!(!should_skip_new_version(None, started_at));
|
||||
assert!(!should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
assert!(should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_outcomes_require_each_requested_endpoint_once_and_ok() {
|
||||
let result = HealResultItem {
|
||||
@@ -1399,10 +1246,8 @@ mod resume_loop_tests {
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
lifecycle_expired: Mutex<HashSet<String>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
fail_listing: AtomicBool,
|
||||
}
|
||||
@@ -1429,15 +1274,9 @@ mod resume_loop_tests {
|
||||
.unwrap()
|
||||
.insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string()));
|
||||
}
|
||||
fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) {
|
||||
self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version));
|
||||
}
|
||||
fn calls(&self) -> Vec<(String, Option<String>)> {
|
||||
self.heal_calls.lock().unwrap().clone()
|
||||
}
|
||||
fn list_include_lifecycle_object_info_calls(&self) -> Vec<bool> {
|
||||
self.list_include_lifecycle_object_info.lock().unwrap().clone()
|
||||
}
|
||||
fn fail_listing(&self) {
|
||||
self.fail_listing.store(true, Ordering::SeqCst);
|
||||
}
|
||||
@@ -1491,23 +1330,6 @@ mod resume_loop_tests {
|
||||
async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test))
|
||||
}
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(self
|
||||
.lifecycle_expired
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains(&compose_key(object, version_id)))
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -1564,12 +1386,7 @@ mod resume_loop_tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_include_lifecycle_object_info
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(include_lifecycle_object_info);
|
||||
if self.fail_listing.load(Ordering::SeqCst) {
|
||||
return Err(Error::other("injected listing failure"));
|
||||
}
|
||||
@@ -1659,7 +1476,6 @@ mod resume_loop_tests {
|
||||
|
||||
/// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result).
|
||||
async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) {
|
||||
let state = env.resume.get_state().await;
|
||||
let mut current_object_index = 0usize;
|
||||
let mut processed = 0u64;
|
||||
let mut successful = 0u64;
|
||||
@@ -1678,7 +1494,6 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result)
|
||||
@@ -1744,7 +1559,6 @@ mod resume_loop_tests {
|
||||
let mut successful = 0;
|
||||
let mut failed = 0;
|
||||
let mut skipped = 0;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
|
||||
let error = healer
|
||||
.heal_bucket_with_resume(
|
||||
@@ -1758,7 +1572,6 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
.expect_err("a remounted target must not begin a new page scan");
|
||||
@@ -1828,109 +1641,6 @@ mod resume_loop_tests {
|
||||
assert_eq!(skipped, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_progress_accumulates_healed_object_bytes() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"first",
|
||||
Some("v1"),
|
||||
HealResultItem {
|
||||
object_size: 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"second",
|
||||
Some("v2"),
|
||||
HealResultItem {
|
||||
object_size: 2048,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 2);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 2);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.bytes_processed, 3072);
|
||||
assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_written_after_heal_started() {
|
||||
let env = make_env().await;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![
|
||||
item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS),
|
||||
item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1),
|
||||
],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_lifecycle_expired("expired", Some("v1"));
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]);
|
||||
assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_ilm_expired, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_listing_failure_does_not_mark_set_completed() {
|
||||
let env = make_env().await;
|
||||
|
||||
+62
-468
@@ -220,11 +220,6 @@ struct CompletedHealStatus {
|
||||
result_items: Vec<HealResultItem>,
|
||||
result_items_truncated: bool,
|
||||
completed_at: SystemTime,
|
||||
/// Sequence-stamped retained window, archived with the completion so
|
||||
/// incremental consumers keep their cursor across the transition (HS-06).
|
||||
seqed_items: Vec<(u64, HealResultItem)>,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -245,65 +240,6 @@ pub struct HealTaskReport {
|
||||
pub result_items: Vec<HealResultItem>,
|
||||
pub result_items_truncated: bool,
|
||||
pub progress: Option<HealProgress>,
|
||||
/// Cursor for incremental consumption: sequence number of the next item
|
||||
/// to be produced. `0` on reports from sources without sequencing.
|
||||
pub next_seq: u64,
|
||||
/// Oldest sequence still retained (`0` together with `next_seq` when
|
||||
/// sequencing is unavailable).
|
||||
pub min_seq: u64,
|
||||
}
|
||||
|
||||
/// Report from a live task, honoring the client's incremental cursor.
|
||||
async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskReport {
|
||||
let window = task.get_result_items_since(since).await;
|
||||
HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
result_items: window.items,
|
||||
// The legacy flag stays set once anything was evicted; a lagging
|
||||
// incremental cursor additionally marks this response truncated so
|
||||
// the client knows to restart from `min_seq`.
|
||||
result_items_truncated: task.result_items_truncated() || window.lagged,
|
||||
progress: Some(task.get_progress().await),
|
||||
next_seq: window.next_seq,
|
||||
min_seq: window.min_seq,
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
|
||||
HealTaskReport {
|
||||
status,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
progress: None,
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) -> HealTaskReport {
|
||||
let mut lagged = false;
|
||||
let result_items = match since {
|
||||
None => completed.result_items.clone(),
|
||||
Some(cursor) => {
|
||||
if cursor + 1 < completed.min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
completed
|
||||
.seqed_items
|
||||
.iter()
|
||||
.filter(|(seq, _)| *seq > cursor)
|
||||
.map(|(_, item)| item.clone())
|
||||
.collect()
|
||||
}
|
||||
};
|
||||
HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items,
|
||||
result_items_truncated: completed.result_items_truncated || lagged,
|
||||
progress: None,
|
||||
next_seq: completed.next_seq,
|
||||
min_seq: completed.min_seq,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
||||
@@ -334,8 +270,6 @@ pub struct HealSourceCounts {
|
||||
pub auto_heal: u64,
|
||||
pub internal: u64,
|
||||
pub read_repair: u64,
|
||||
#[serde(default)]
|
||||
pub mrf: u64,
|
||||
}
|
||||
|
||||
impl HealSourceCounts {
|
||||
@@ -346,7 +280,6 @@ impl HealSourceCounts {
|
||||
HealRequestSource::AutoHeal => self.auto_heal += 1,
|
||||
HealRequestSource::Internal => self.internal += 1,
|
||||
HealRequestSource::ReadRepair => self.read_repair += 1,
|
||||
HealRequestSource::Mrf => self.mrf += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -595,11 +528,6 @@ impl PriorityHealQueue {
|
||||
self.dedup_keys.contains_key(&key)
|
||||
}
|
||||
|
||||
/// Iterate queued requests (used by the admin overlap check).
|
||||
fn requests(&self) -> impl Iterator<Item = &HealRequest> {
|
||||
self.heap.iter().map(|item| &item.request)
|
||||
}
|
||||
|
||||
fn contains_request_id(&self, request_id: &str) -> bool {
|
||||
self.heap.iter().any(|item| item.request.id == request_id)
|
||||
}
|
||||
@@ -758,80 +686,6 @@ fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
|
||||
}
|
||||
|
||||
/// Heal config
|
||||
/// HS-06 admin overlap policy.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum HealOverlapPolicy {
|
||||
/// Default: overlapping admin starts merge into the existing task
|
||||
/// (today's dedup semantics).
|
||||
#[default]
|
||||
Merge,
|
||||
/// Return a typed already-running / overlapping-paths rejection like
|
||||
/// madmin's ErrHealAlreadyRunning / ErrHealOverlappingPaths.
|
||||
MinioError,
|
||||
}
|
||||
|
||||
/// Path view of a heal type for overlap comparison: a bucket plus a
|
||||
/// prefix/object path inside it (`None` bucket = cluster-wide, overlaps
|
||||
/// everything).
|
||||
fn heal_type_path_view(heal_type: &HealType) -> (Option<&str>, &str) {
|
||||
match heal_type {
|
||||
HealType::Cluster => (None, ""),
|
||||
HealType::Bucket { bucket } => (Some(bucket), ""),
|
||||
HealType::Prefix { bucket, prefix } => (Some(bucket), prefix),
|
||||
HealType::Object { bucket, object, .. }
|
||||
| HealType::Metadata { bucket, object }
|
||||
| HealType::ECDecode { bucket, object, .. } => (Some(bucket), object),
|
||||
// MRF/MetaPath heal keys on a meta path; treat the whole set of
|
||||
// buckets as one namespace so it only overlaps itself exactly.
|
||||
HealType::MRF { meta_path } => (Some("\u{0}mrf"), meta_path),
|
||||
// Erasure-set heal: the set id is the overlap dimension.
|
||||
HealType::ErasureSet { set_disk_id, .. } => (Some("\u{0}set"), set_disk_id),
|
||||
}
|
||||
}
|
||||
|
||||
/// How two heal paths relate for the admin overlap check (HS-06).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum OverlapVerdict {
|
||||
/// Distinct targets: no conflict.
|
||||
Disjoint,
|
||||
/// Same target: an identical heal is already in flight.
|
||||
SameTarget,
|
||||
/// One target contains the other.
|
||||
Overlapping,
|
||||
}
|
||||
|
||||
fn prefix_paths_overlap(a: &str, b: &str) -> OverlapVerdict {
|
||||
if a == b {
|
||||
return OverlapVerdict::SameTarget;
|
||||
}
|
||||
if a.is_empty() || b.is_empty() || a.starts_with(b) || b.starts_with(a) {
|
||||
return OverlapVerdict::Overlapping;
|
||||
}
|
||||
OverlapVerdict::Disjoint
|
||||
}
|
||||
|
||||
fn heal_types_overlap(left: &HealType, right: &HealType) -> OverlapVerdict {
|
||||
let (left_bucket, left_path) = heal_type_path_view(left);
|
||||
let (right_bucket, right_path) = heal_type_path_view(right);
|
||||
match (left_bucket, right_bucket) {
|
||||
// Cluster-wide overlaps everything (but an exact cluster match is
|
||||
// SameTarget).
|
||||
(None, _) | (_, None) => {
|
||||
if matches!(left, HealType::Cluster) && matches!(right, HealType::Cluster) {
|
||||
OverlapVerdict::SameTarget
|
||||
} else {
|
||||
OverlapVerdict::Overlapping
|
||||
}
|
||||
}
|
||||
(Some(lb), Some(rb)) => {
|
||||
if lb != rb {
|
||||
return OverlapVerdict::Disjoint;
|
||||
}
|
||||
prefix_paths_overlap(left_path, right_path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealConfig {
|
||||
/// Whether to enable auto heal
|
||||
@@ -852,9 +706,6 @@ pub struct HealConfig {
|
||||
pub low_priority_drop_when_full: bool,
|
||||
/// Whether notify-driven scheduler wakeups are enabled.
|
||||
pub event_driven_scheduler_enable: bool,
|
||||
/// How admin heal starts behave on path overlap (HS-06): merge into the
|
||||
/// existing task (default) or return a typed already-running rejection.
|
||||
pub overlap_policy: HealOverlapPolicy,
|
||||
/// Whether per-set bulkhead scheduling is enabled.
|
||||
pub set_bulkhead_enable: bool,
|
||||
/// Whether erasure-set page parallelism is enabled.
|
||||
@@ -903,14 +754,6 @@ impl Default for HealConfig {
|
||||
rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
|
||||
rustfs_config::DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
|
||||
);
|
||||
let overlap_policy =
|
||||
match rustfs_utils::get_env_str(rustfs_config::ENV_HEAL_OVERLAP_POLICY, rustfs_config::DEFAULT_HEAL_OVERLAP_POLICY)
|
||||
.to_lowercase()
|
||||
.as_str()
|
||||
{
|
||||
"minio_error" => HealOverlapPolicy::MinioError,
|
||||
_ => HealOverlapPolicy::Merge,
|
||||
};
|
||||
let set_bulkhead_enable = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE,
|
||||
rustfs_config::DEFAULT_HEAL_SET_BULKHEAD_ENABLE,
|
||||
@@ -947,7 +790,6 @@ impl Default for HealConfig {
|
||||
low_priority_merge_enable,
|
||||
low_priority_drop_when_full,
|
||||
event_driven_scheduler_enable,
|
||||
overlap_policy,
|
||||
set_bulkhead_enable,
|
||||
page_parallel_enable,
|
||||
mainline_throttle_enable,
|
||||
@@ -1914,50 +1756,6 @@ impl HealManager {
|
||||
request: HealRequest,
|
||||
preserve_alias: bool,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
// HS-06 forceStart semantics (admin only): MinIO stops the old task
|
||||
// first and then starts the new one. Cancel any active admin task
|
||||
// overlapping this request's path before entering admission, so the
|
||||
// fresh task is never merged into the one being replaced.
|
||||
if request.source == HealRequestSource::Admin && request.force_start {
|
||||
let overlapping: Vec<String> = {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
active_heals
|
||||
.iter()
|
||||
.filter(|(task_id, task)| {
|
||||
task.source == HealRequestSource::Admin
|
||||
&& heal_types_overlap(&request.heal_type, &task.heal_type) != OverlapVerdict::Disjoint
|
||||
&& *task_id != &request.id
|
||||
})
|
||||
.map(|(task_id, _)| task_id.clone())
|
||||
.collect()
|
||||
};
|
||||
for task_id in overlapping {
|
||||
match self.cancel_task(&task_id).await {
|
||||
Ok(_) => info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
cancelled_task_id = %task_id,
|
||||
result = "force_start_cancelled_overlap",
|
||||
"Admin forceStart cancelled an overlapping heal task"
|
||||
),
|
||||
Err(err) => warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
cancelled_task_id = %task_id,
|
||||
error = %err,
|
||||
result = "force_start_cancel_failed",
|
||||
"Admin forceStart failed to cancel an overlapping heal task"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let config = self.config.read().await;
|
||||
let dedup_key = PriorityHealQueue::make_dedup_key(&request);
|
||||
|
||||
@@ -1980,15 +1778,7 @@ impl HealManager {
|
||||
.or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying")))
|
||||
});
|
||||
if let Some((merged_task_id, duplicate_state)) = duplicate.flatten() {
|
||||
// HS-06: under the minio_error overlap policy an exact duplicate
|
||||
// admin start reports the typed AlreadyRunning rejection instead
|
||||
// of the silent merge (MinIO's ErrHealAlreadyRunning).
|
||||
let admission =
|
||||
if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
} else {
|
||||
Self::duplicate_admission_for_request(&request, &config)
|
||||
};
|
||||
let admission = Self::duplicate_admission_for_request(&request, &config);
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
@@ -2034,62 +1824,6 @@ impl HealManager {
|
||||
});
|
||||
}
|
||||
|
||||
// HS-06 typed overlap rejection (admin only, minio_error policy):
|
||||
// paths containing or contained by an active/queued task reject with
|
||||
// AlreadyRunning / OverlappingPaths instead of merging. Exact
|
||||
// duplicates already merged above; scanner/autoheal/read-repair
|
||||
// sources never take this path.
|
||||
if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError {
|
||||
let mut rejection = None;
|
||||
for (task_id, task) in active_heals.iter() {
|
||||
match heal_types_overlap(&request.heal_type, &task.heal_type) {
|
||||
OverlapVerdict::SameTarget => {
|
||||
rejection = Some((HealAdmissionDropReason::AlreadyRunning, task_id.clone()));
|
||||
break;
|
||||
}
|
||||
OverlapVerdict::Overlapping => {
|
||||
rejection = Some((HealAdmissionDropReason::OverlappingPaths, task_id.clone()));
|
||||
}
|
||||
OverlapVerdict::Disjoint => {}
|
||||
}
|
||||
}
|
||||
if rejection.is_none() {
|
||||
for queued in queue.requests() {
|
||||
match heal_types_overlap(&request.heal_type, &queued.heal_type) {
|
||||
OverlapVerdict::SameTarget => {
|
||||
rejection = Some((HealAdmissionDropReason::AlreadyRunning, queued.id.clone()));
|
||||
break;
|
||||
}
|
||||
OverlapVerdict::Overlapping => {
|
||||
rejection = Some((HealAdmissionDropReason::OverlappingPaths, queued.id.clone()));
|
||||
}
|
||||
OverlapVerdict::Disjoint => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((reason, overlap_task_id)) = rejection {
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Dropped(reason), "overlap_rejected");
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
overlap_task_id = %overlap_task_id,
|
||||
reason = reason.as_str(),
|
||||
result = "overlap_rejected",
|
||||
"Admin heal start rejected by overlap policy"
|
||||
);
|
||||
return Ok(HealAdmissionReceipt {
|
||||
result: HealAdmissionResult::Dropped(reason),
|
||||
task_id: overlap_task_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut task_id = request.id.clone();
|
||||
let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
|
||||
if admission == HealAdmissionResult::Merged
|
||||
@@ -2162,25 +1896,28 @@ impl HealManager {
|
||||
}
|
||||
|
||||
pub async fn get_task_report(&self, task_id: &str) -> Result<HealTaskReport> {
|
||||
self.get_task_report_since(task_id, None).await
|
||||
}
|
||||
|
||||
/// Incremental variant of [`Self::get_task_report`] (HS-06): `since` is
|
||||
/// the client's last seen sequence number; `None` keeps the legacy
|
||||
/// full-snapshot semantics.
|
||||
pub async fn get_task_report_since(&self, task_id: &str, since: Option<u64>) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
{
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
return Ok(active_task_report(task, since).await);
|
||||
return Ok(HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
result_items: task.get_result_items().await,
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
progress: Some(task.get_progress().await),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
|
||||
return Ok(empty_task_report(retrying.status()));
|
||||
return Ok(HealTaskReport {
|
||||
status: retrying.status(),
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2190,21 +1927,36 @@ impl HealManager {
|
||||
if let Some(completed) = completed_heals.get(&canonical_task_id)
|
||||
&& completed_status_is_retrying(&completed.status)
|
||||
{
|
||||
return Ok(completed_task_report(completed, since));
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
result_items_truncated: completed.result_items_truncated,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let queue = self.heal_queue.lock().await;
|
||||
if queue.contains_request_id(&canonical_task_id) {
|
||||
return Ok(empty_task_report(HealTaskStatus::Pending));
|
||||
return Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut completed_heals = self.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
if let Some(completed) = completed_heals.get(&canonical_task_id) {
|
||||
return Ok(completed_task_report(completed, since));
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
result_items_truncated: completed.result_items_truncated,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
|
||||
Err(Error::TaskNotFound {
|
||||
@@ -2213,23 +1965,18 @@ impl HealManager {
|
||||
}
|
||||
|
||||
pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskReport> {
|
||||
self.get_task_report_for_path_since(heal_path, task_id, None).await
|
||||
}
|
||||
|
||||
/// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
|
||||
pub async fn get_task_report_for_path_since(
|
||||
&self,
|
||||
heal_path: &str,
|
||||
task_id: &str,
|
||||
since: Option<u64>,
|
||||
) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
{
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id)
|
||||
&& heal_type_matches_path(&task.heal_type, heal_path)
|
||||
{
|
||||
return Ok(active_task_report(task, since).await);
|
||||
return Ok(HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
result_items: task.get_result_items().await,
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
progress: Some(task.get_progress().await),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2238,7 +1985,12 @@ impl HealManager {
|
||||
if let Some(retrying) = retrying_heals.get(&canonical_task_id)
|
||||
&& heal_type_matches_path(&retrying.request.heal_type, heal_path)
|
||||
{
|
||||
return Ok(empty_task_report(retrying.status()));
|
||||
return Ok(HealTaskReport {
|
||||
status: retrying.status(),
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2249,14 +2001,24 @@ impl HealManager {
|
||||
&& heal_type_matches_path(&completed.heal_type, heal_path)
|
||||
&& completed_status_is_retrying(&completed.status)
|
||||
{
|
||||
return Ok(completed_task_report(completed, since));
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
result_items_truncated: completed.result_items_truncated,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let queue = self.heal_queue.lock().await;
|
||||
if queue.contains_request_id_matching_path(&canonical_task_id, heal_path) {
|
||||
return Ok(empty_task_report(HealTaskStatus::Pending));
|
||||
return Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2266,7 +2028,12 @@ impl HealManager {
|
||||
if let Some(completed) = completed_heals.get(&canonical_task_id)
|
||||
&& heal_type_matches_path(&completed.heal_type, heal_path)
|
||||
{
|
||||
return Ok(completed_task_report(completed, since));
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
result_items_truncated: completed.result_items_truncated,
|
||||
progress: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2618,27 +2385,8 @@ impl HealManager {
|
||||
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
|
||||
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
|
||||
snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
|
||||
snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
|
||||
snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
|
||||
snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
|
||||
snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
|
||||
snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
|
||||
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
|
||||
(Some(current), Some(next)) => Some(current.min(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
|
||||
(Some(current), Some(next)) => Some(current.max(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
if progress.current_object.is_some() {
|
||||
snapshot.current_object = progress.current_object;
|
||||
}
|
||||
}
|
||||
snapshot.refresh_progress_percentage();
|
||||
snapshot.refresh_estimated_completion_time();
|
||||
Some(snapshot)
|
||||
}
|
||||
|
||||
@@ -3460,17 +3208,12 @@ impl HealManager {
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
let final_window = completed_task.get_result_items_since(None).await;
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
result_items: final_window.items.clone(),
|
||||
result_items: completed_task.get_result_items().await,
|
||||
result_items_truncated: completed_task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items: completed_task.get_seqed_result_items().await,
|
||||
next_seq: final_window.next_seq,
|
||||
min_seq: final_window.min_seq,
|
||||
};
|
||||
let mut completed_heals_guard = completed_heals_clone.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals_guard);
|
||||
@@ -3480,7 +3223,6 @@ impl HealManager {
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
|
||||
}
|
||||
HealTaskStatus::Retrying { .. } => {}
|
||||
_ => {
|
||||
@@ -4007,7 +3749,6 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
@@ -5242,9 +4983,6 @@ mod tests {
|
||||
},
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
);
|
||||
@@ -5526,136 +5264,6 @@ mod tests {
|
||||
assert_eq!(snapshot.queued_by_source.internal, 0);
|
||||
}
|
||||
|
||||
// HS-06 (backlog#1870): overlap policy + forceStart semantics.
|
||||
fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
overlap_policy: policy,
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Prefix {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: prefix.to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
request
|
||||
}
|
||||
|
||||
async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String {
|
||||
let task = Arc::new(HealTask::from_request(request, manager.storage.clone()));
|
||||
let task_id = task.id.clone();
|
||||
manager.active_heals.lock().await.insert(task_id.clone(), task);
|
||||
task_id
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlap_policy_minio_error_rejects_same_and_containing_paths() {
|
||||
let manager = manager_with_policy(HealOverlapPolicy::MinioError);
|
||||
insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;
|
||||
|
||||
// Same target: typed AlreadyRunning.
|
||||
let same = manager
|
||||
.submit_heal_request(admin_prefix_request("bucket-a", "logs/"))
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(
|
||||
same,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning),
|
||||
"an identical target must reject with already-running"
|
||||
);
|
||||
|
||||
// Contained path: typed OverlappingPaths.
|
||||
let nested = manager
|
||||
.submit_heal_request(admin_prefix_request("bucket-a", "logs/app/"))
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(
|
||||
nested,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths),
|
||||
"a path inside the active task's path must reject with overlapping-paths"
|
||||
);
|
||||
|
||||
// Containing path (bucket-wide vs nested active): also overlapping.
|
||||
let wide = manager
|
||||
.submit_heal_request(admin_prefix_request("bucket-a", ""))
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(
|
||||
wide,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths),
|
||||
"a bucket-wide start overlapping a nested active heal must reject"
|
||||
);
|
||||
|
||||
// Disjoint bucket: unaffected.
|
||||
let disjoint = manager
|
||||
.submit_heal_request(admin_prefix_request("bucket-b", "logs/"))
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(disjoint, HealAdmissionResult::Accepted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overlap_policy_default_merge_keeps_today_semantics() {
|
||||
let manager = manager_with_policy(HealOverlapPolicy::Merge);
|
||||
insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;
|
||||
|
||||
// Different-dedup-key overlap still merges under the default policy:
|
||||
// the nested path dedups to its own key but nothing rejects it.
|
||||
let nested = manager
|
||||
.submit_heal_request(admin_prefix_request("bucket-a", "logs/app/"))
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(nested, HealAdmissionResult::Accepted, "default policy must not reject overlaps");
|
||||
|
||||
// Non-admin sources never get overlap rejections even under minio_error.
|
||||
let manager = manager_with_policy(HealOverlapPolicy::MinioError);
|
||||
insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;
|
||||
let mut scanner_request = admin_prefix_request("bucket-a", "logs/app/");
|
||||
scanner_request.source = HealRequestSource::Scanner;
|
||||
let admitted = manager
|
||||
.submit_heal_request(scanner_request)
|
||||
.await
|
||||
.expect("admission must decide");
|
||||
assert_eq!(admitted, HealAdmissionResult::Accepted, "scanner sources must never be overlap-rejected");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_force_start_cancels_overlapping_active_task_first() {
|
||||
let manager = manager_with_policy(HealOverlapPolicy::Merge);
|
||||
let old_id = insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;
|
||||
|
||||
let mut replacement = admin_prefix_request("bucket-a", "logs/");
|
||||
replacement.force_start = true;
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(replacement)
|
||||
.await
|
||||
.expect("force-start submission must decide");
|
||||
|
||||
assert!(receipt.result.is_admitted(), "the new task must be admitted (Accepted or Merged)");
|
||||
let old_task_gone = {
|
||||
let active_heals = manager.active_heals.lock().await;
|
||||
!active_heals.contains_key(&old_id)
|
||||
};
|
||||
assert!(
|
||||
old_task_gone,
|
||||
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
|
||||
);
|
||||
assert!(
|
||||
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
|
||||
"a cancelled task must no longer resolve as an active heal"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_operations_snapshot_counts_active_by_source_and_priority() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -5788,8 +5396,6 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = first.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(20));
|
||||
progress.set_total_baseline(12, 8192);
|
||||
progress.update_progress(7, 3, 1, 4096);
|
||||
}
|
||||
|
||||
@@ -5799,8 +5405,6 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = second.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(8, 4096);
|
||||
progress.update_progress(11, 5, 2, 2048);
|
||||
}
|
||||
|
||||
@@ -5815,11 +5419,7 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 18);
|
||||
assert_eq!(progress.objects_healed, 8);
|
||||
assert_eq!(progress.objects_failed, 3);
|
||||
assert_eq!(progress.objects_total_count, 20);
|
||||
assert_eq!(progress.objects_total_size, 12288);
|
||||
assert_eq!(progress.bytes_processed, 6144);
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -5958,9 +5558,6 @@ mod tests {
|
||||
status: HealTaskStatus::Completed,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
);
|
||||
@@ -5995,9 +5592,6 @@ mod tests {
|
||||
..Default::default()
|
||||
}],
|
||||
result_items_truncated: true,
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -16,7 +16,6 @@ pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod event;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod progress;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
|
||||
@@ -1,682 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Mission Repair Feed (MRF) queue, journal, and consumer.
|
||||
//!
|
||||
//! Intents arriving on the global channel (see `rustfs_common::mrf_channel`)
|
||||
//! are buffered in a bounded in-memory queue, translated into prioritized
|
||||
//! heal requests, and — while they are not yet accepted by the heal manager —
|
||||
//! mirrored into a durable journal so a crash or restart can replay them.
|
||||
//! This is the RustFS counterpart of MinIO's `.heal/mrf/list.bin` replay,
|
||||
//! layered on top of (not replacing) read-repair and scanner heal.
|
||||
//!
|
||||
//! Durability model: the journal is a snapshot of the *unaccepted* pending
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed
|
||||
//! duplicates are merged by the manager's dedup key, and read-repair remains
|
||||
//! the safety net.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::HealManager;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
|
||||
/// Journal location inside the metadata bucket, following the resume-state
|
||||
/// layout.
|
||||
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
|
||||
|
||||
/// Record format tag.
|
||||
const MRF_JOURNAL_FORMAT: u8 = 1;
|
||||
/// Record layout version.
|
||||
const MRF_JOURNAL_VERSION: u8 = 1;
|
||||
|
||||
/// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
|
||||
/// has_version flag.
|
||||
const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct MrfConsumerConfig {
|
||||
/// In-memory queue capacity in intents.
|
||||
pub queue_capacity: usize,
|
||||
/// Journal byte budget; a pending snapshot above this bound is rejected
|
||||
/// oldest-first so the journal can never grow unbounded.
|
||||
pub journal_max_bytes: usize,
|
||||
/// How many journal intents to re-arm per replay round.
|
||||
pub replay_batch: usize,
|
||||
/// Group-commit cadence for the journal snapshot.
|
||||
pub flush_interval: Duration,
|
||||
/// New intents between flushes that force an early snapshot.
|
||||
pub flush_threshold: usize,
|
||||
/// Backoff after the heal manager reports a full admission.
|
||||
pub admission_backoff: Duration,
|
||||
}
|
||||
|
||||
impl Default for MrfConsumerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
queue_capacity: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_QUEUE_SIZE,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_QUEUE_SIZE,
|
||||
),
|
||||
journal_max_bytes: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_JOURNAL_MAX_BYTES,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES,
|
||||
),
|
||||
replay_batch: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_REPLAY_BATCH,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_REPLAY_BATCH,
|
||||
),
|
||||
flush_interval: Duration::from_millis(500),
|
||||
flush_threshold: 1000,
|
||||
admission_backoff: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded pending set with count and byte ceilings. Overflow drops the
|
||||
/// incoming intent (never a resident one) and counts the loss.
|
||||
pub(crate) struct MrfQueue {
|
||||
pending: VecDeque<MrfIntent>,
|
||||
bytes: usize,
|
||||
capacity: usize,
|
||||
byte_budget: usize,
|
||||
}
|
||||
|
||||
impl MrfQueue {
|
||||
pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
|
||||
Self {
|
||||
pending: VecDeque::new(),
|
||||
bytes: 0,
|
||||
capacity,
|
||||
byte_budget,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `false` (after counting) when either ceiling would be crossed.
|
||||
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
|
||||
let cost = intent.estimated_bytes();
|
||||
if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
return false;
|
||||
}
|
||||
self.bytes += cost;
|
||||
self.pending.push_back(intent);
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
|
||||
let intent = self.pending.pop_front()?;
|
||||
self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
|
||||
Some(intent)
|
||||
}
|
||||
|
||||
pub(crate) fn push_back(&mut self, intent: MrfIntent) {
|
||||
self.bytes += intent.estimated_bytes();
|
||||
self.pending.push_back(intent);
|
||||
}
|
||||
|
||||
pub(crate) fn depth(&self) -> usize {
|
||||
self.pending.len()
|
||||
}
|
||||
|
||||
pub(crate) fn bytes(&self) -> usize {
|
||||
self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn intents(&self) -> impl Iterator<Item = &MrfIntent> {
|
||||
self.pending.iter()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Journal record codec
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Append one encoded record to `out`.
|
||||
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
|
||||
let start = out.len();
|
||||
out.push(MRF_JOURNAL_FORMAT);
|
||||
out.push(MRF_JOURNAL_VERSION);
|
||||
out.push(match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite => 3,
|
||||
});
|
||||
out.push(intent.attempts);
|
||||
out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
|
||||
match intent.version_id {
|
||||
Some(bytes) => {
|
||||
out.push(1);
|
||||
out.extend_from_slice(&bytes);
|
||||
}
|
||||
None => out.push(0),
|
||||
}
|
||||
out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(intent.bucket.as_bytes());
|
||||
out.extend_from_slice(intent.object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&out[start..]);
|
||||
out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
|
||||
}
|
||||
|
||||
fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
|
||||
return None;
|
||||
}
|
||||
if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
|
||||
return None;
|
||||
}
|
||||
let kind = match data[2] {
|
||||
1 => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
2 => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
|
||||
3 => rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
_ => return None,
|
||||
};
|
||||
let attempts = data[3];
|
||||
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
|
||||
let has_version = data[12] != 0;
|
||||
let mut cursor = MRF_RECORD_FIXED_HEAD;
|
||||
let version_id = if has_version {
|
||||
if data.len() < cursor + 16 {
|
||||
return None;
|
||||
}
|
||||
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
|
||||
cursor += 16;
|
||||
Some(bytes)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if data.len() < cursor + 8 {
|
||||
return None;
|
||||
}
|
||||
let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
|
||||
let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
|
||||
cursor += 8;
|
||||
let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
|
||||
let record_end = body_end.checked_add(4)?;
|
||||
if data.len() < record_end {
|
||||
return None;
|
||||
}
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&data[..body_end]);
|
||||
if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
|
||||
return None;
|
||||
}
|
||||
let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
|
||||
let object = std::sync::Arc::from(std::str::from_utf8(&data[cursor + bucket_len..body_end]).ok()?);
|
||||
Some((
|
||||
MrfIntent {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
kind,
|
||||
enqueued_at_ms,
|
||||
attempts,
|
||||
},
|
||||
record_end,
|
||||
))
|
||||
}
|
||||
|
||||
/// Decode a whole journal, stopping at the first torn or corrupt record.
|
||||
/// Returns the decoded intents and the number of trailing bytes discarded.
|
||||
pub(crate) fn decode_journal(data: &[u8]) -> (Vec<MrfIntent>, usize) {
|
||||
let mut intents = Vec::new();
|
||||
let mut cursor = 0usize;
|
||||
while cursor < data.len() {
|
||||
match decode_one(&data[cursor..]) {
|
||||
Some((intent, consumed)) => {
|
||||
intents.push(intent);
|
||||
cursor += consumed;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let truncated = data.len() - cursor;
|
||||
(intents, truncated)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Journal disk IO (all local disks, first successful read wins)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn journal_disks() -> Vec<DiskStore> {
|
||||
let map = local_disk_map_read().await;
|
||||
map.values().flatten().cloned().collect()
|
||||
}
|
||||
|
||||
async fn read_journal() -> Option<Vec<u8>> {
|
||||
for disk in journal_disks().await {
|
||||
match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
|
||||
Ok(bytes) => return Some(bytes.to_vec()),
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn write_journal(data: &[u8]) {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
for disk in journal_disks().await {
|
||||
if let Err(err) = disk
|
||||
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
|
||||
.await
|
||||
{
|
||||
warn_mrf_journal_write(&err);
|
||||
}
|
||||
}
|
||||
if !data.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
|
||||
}
|
||||
|
||||
async fn delete_journal() {
|
||||
for disk in journal_disks().await {
|
||||
let _ = disk
|
||||
.delete(
|
||||
super::RUSTFS_META_BUCKET,
|
||||
MRF_JOURNAL_PATH,
|
||||
crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_mrf_journal_write(err: &super::DiskError) {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = %err,
|
||||
"MRF journal write failed; unconsumed intents may be lost on restart"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Consumer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Translate an intent into the prioritized heal request the issue specifies:
|
||||
/// decode failures go Urgent ECDecode, metadata corruption goes High
|
||||
/// Metadata, partial writes go Normal object heal.
|
||||
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
let bucket = intent.bucket.to_string();
|
||||
let object = intent.object.to_string();
|
||||
let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
|
||||
let (heal_type, priority) = match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
HealPriority::Urgent,
|
||||
),
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => (HealType::Metadata { bucket, object }, HealPriority::High),
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite => (
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
};
|
||||
let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
|
||||
request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
|
||||
request
|
||||
}
|
||||
|
||||
struct MrfRuntime {
|
||||
queue: MrfQueue,
|
||||
config: MrfConsumerConfig,
|
||||
new_since_flush: usize,
|
||||
/// True while a journal snapshot exists on disk that no longer reflects
|
||||
/// an all-consumed pending set; the next idle tick removes it (MinIO
|
||||
/// deletes its `list.bin` after replay for the same reason).
|
||||
journal_on_disk: bool,
|
||||
/// Earliest instant a full-admission retry may proceed.
|
||||
backoff_until: Option<tokio::time::Instant>,
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn record_accept(&mut self) {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot, which is the journal's compaction.
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
encode_intent(intent, &mut buf);
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
write_journal(&self.snapshot()).await;
|
||||
self.new_since_flush = 0;
|
||||
self.journal_on_disk = true;
|
||||
}
|
||||
|
||||
/// Drain pending intents into the heal manager until it is full, the
|
||||
/// queue empties, or attempts are exhausted.
|
||||
async fn dispatch(&mut self, manager: &HealManager) {
|
||||
if let Some(until) = self.backoff_until {
|
||||
if tokio::time::Instant::now() < until {
|
||||
return;
|
||||
}
|
||||
self.backoff_until = None;
|
||||
}
|
||||
while let Some(mut intent) = self.queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(),
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff);
|
||||
break;
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) => {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
|
||||
}
|
||||
Err(_) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
|
||||
gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and
|
||||
/// spawn the consumer task. Called once from the heal runtime bootstrap right
|
||||
/// after the manager started; a disabled feature or a double call is a no-op.
|
||||
/// Public for integration tests that drive the real consumer loop.
|
||||
pub fn spawn_mrf_consumer(manager: Arc<HealManager>) {
|
||||
let enabled = rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_MRF_ENABLE, rustfs_config::DEFAULT_HEAL_MRF_ENABLE);
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(enabled);
|
||||
if !enabled {
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF intent pipeline disabled by configuration; producers will not deliver"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let receiver = match rustfs_common::mrf_channel::init_mrf_channel() {
|
||||
Ok(receiver) => receiver,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = err,
|
||||
"MRF channel initialization failed; intents will be dropped at producers"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
run_mrf_consumer(manager, receiver).await;
|
||||
});
|
||||
tracing::info!(target: "rustfs::heal::mrf", "MRF intent consumer started");
|
||||
}
|
||||
|
||||
/// Replay the durable journal into a fresh pending queue and submit whatever
|
||||
/// it armed. Returns the number of intact intents replayed. Duplicates are
|
||||
/// merged by the manager's dedup key; the journal file is removed once read
|
||||
/// (torn tails truncate via the per-record CRC). Public for integration tests;
|
||||
/// the live consumer invokes this through [`replay_into`] at startup.
|
||||
pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
let mut backoff_until: Option<tokio::time::Instant> = None;
|
||||
replay_into(manager, &mut queue, &mut backoff_until).await
|
||||
}
|
||||
|
||||
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
|
||||
async fn replay_into(
|
||||
manager: &Arc<HealManager>,
|
||||
queue: &mut MrfQueue,
|
||||
backoff_until: &mut Option<tokio::time::Instant>,
|
||||
) -> usize {
|
||||
let Some(data) = read_journal().await else {
|
||||
return 0;
|
||||
};
|
||||
let (intents, truncated) = decode_journal(&data);
|
||||
if truncated > 0 {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
truncated_bytes = truncated,
|
||||
"MRF journal had a torn tail; truncated records were discarded"
|
||||
);
|
||||
}
|
||||
counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
|
||||
let replayed = intents.len();
|
||||
for intent in intents {
|
||||
queue.try_push(intent);
|
||||
}
|
||||
delete_journal().await;
|
||||
|
||||
// Drain the replayed intents immediately; whatever the manager refuses
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
if backoff_until.is_none() {
|
||||
while let Some(mut intent) = queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts < MRF_MAX_ATTEMPTS {
|
||||
queue.push_back(intent);
|
||||
*backoff_until = Some(tokio::time::Instant::now());
|
||||
}
|
||||
break;
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) | Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
replayed
|
||||
}
|
||||
|
||||
/// Replay the journal, then keep draining the channel into the heal manager
|
||||
/// while persisting the pending snapshot.
|
||||
async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receiver<MrfIntent>) {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
||||
config: config.clone(),
|
||||
new_since_flush: 0,
|
||||
journal_on_disk: false,
|
||||
backoff_until: None,
|
||||
};
|
||||
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
|
||||
let mut flush_tick = tokio::time::interval(runtime.config.flush_interval);
|
||||
flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
let mut batch: Vec<MrfIntent> = Vec::with_capacity(runtime.config.replay_batch);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
// Channel closed: flush once more and stop.
|
||||
runtime.flush().await;
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF channel closed; consumer stopped after final flush"
|
||||
);
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
runtime.queue.try_push(intent);
|
||||
runtime.new_since_flush += 1;
|
||||
}
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
if runtime.new_since_flush >= runtime.config.flush_threshold {
|
||||
runtime.flush().await;
|
||||
}
|
||||
}
|
||||
_ = flush_tick.tick() => {
|
||||
if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
} else if runtime.journal_on_disk {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind};
|
||||
use std::sync::Arc as StdArc;
|
||||
|
||||
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
||||
MrfIntent {
|
||||
bucket: StdArc::from(bucket),
|
||||
object: StdArc::from(object),
|
||||
version_id: Some([7u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
enqueued_at_ms: 1_700_000_000_000,
|
||||
attempts,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
|
||||
|
||||
let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
|
||||
assert!(tiny.try_push(intent("bucket", "object", 0)));
|
||||
assert!(
|
||||
!tiny.try_push(intent("bucket", "object", 0)),
|
||||
"byte budget must drop before the second intent fits"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_roundtrip_preserves_intents() {
|
||||
let intents = vec![
|
||||
intent("bucket-a", "object/a", 0),
|
||||
intent("bucket-b", "object/b", 2),
|
||||
MrfIntent {
|
||||
bucket: StdArc::from("bucket-c"),
|
||||
object: StdArc::from("object/c"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
enqueued_at_ms: 5,
|
||||
attempts: 1,
|
||||
},
|
||||
];
|
||||
let mut buf = Vec::new();
|
||||
for intent in &intents {
|
||||
encode_intent(intent, &mut buf);
|
||||
}
|
||||
let (decoded, truncated) = decode_journal(&buf);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(decoded.len(), intents.len());
|
||||
for (left, right) in decoded.iter().zip(intents.iter()) {
|
||||
assert_eq!(left.bucket, right.bucket);
|
||||
assert_eq!(left.object, right.object);
|
||||
assert_eq!(left.version_id, right.version_id);
|
||||
assert_eq!(left.kind, right.kind);
|
||||
assert_eq!(left.attempts, right.attempts);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_torn_tail_is_truncated() {
|
||||
let mut buf = Vec::new();
|
||||
encode_intent(&intent("b", "o", 0), &mut buf);
|
||||
let mut torn = buf.clone();
|
||||
torn.extend_from_slice(&buf[..buf.len() / 2]);
|
||||
|
||||
let (decoded, truncated) = decode_journal(&torn);
|
||||
assert_eq!(decoded.len(), 1, "the intact record must survive");
|
||||
assert!(truncated > 0, "the partial tail must be discarded");
|
||||
|
||||
// A corrupted body (CRC mismatch) also truncates from that record on.
|
||||
let mut corrupt = buf.clone();
|
||||
let mid = MRF_RECORD_FIXED_HEAD + 4;
|
||||
corrupt[mid] ^= 0xff;
|
||||
let (decoded, truncated) = decode_journal(&corrupt);
|
||||
assert!(decoded.is_empty());
|
||||
assert_eq!(truncated, corrupt.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_request_mapping_follows_priority_matrix() {
|
||||
let decode = build_heal_request(&intent("b", "o", 0));
|
||||
assert!(matches!(decode.heal_type, HealType::ECDecode { .. }));
|
||||
assert_eq!(decode.priority, HealPriority::Urgent);
|
||||
|
||||
let metadata = build_heal_request(&MrfIntent {
|
||||
bucket: StdArc::from("b"),
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
assert!(matches!(metadata.heal_type, HealType::Metadata { .. }));
|
||||
assert_eq!(metadata.priority, HealPriority::High);
|
||||
|
||||
let partial = build_heal_request(&MrfIntent {
|
||||
bucket: StdArc::from("b"),
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
assert!(matches!(partial.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(partial.priority, HealPriority::Normal);
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -24,14 +24,6 @@ pub struct HealProgress {
|
||||
pub objects_healed: u64,
|
||||
/// Objects failed
|
||||
pub objects_failed: u64,
|
||||
/// Versions skipped because they were written after this heal started
|
||||
pub skipped_new_versions: u64,
|
||||
/// Versions skipped because lifecycle already selected them for expiry
|
||||
pub skipped_ilm_expired: u64,
|
||||
/// Baseline object count from the latest complete usage snapshot
|
||||
pub objects_total_count: u64,
|
||||
/// Baseline object bytes from the latest complete usage snapshot
|
||||
pub objects_total_size: u64,
|
||||
/// Bytes processed
|
||||
pub bytes_processed: u64,
|
||||
/// Current object
|
||||
@@ -62,56 +54,10 @@ impl HealProgress {
|
||||
self.bytes_processed = bytes;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
|
||||
self.objects_total_count = objects_total_count;
|
||||
self.objects_total_size = objects_total_size;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_new_version(&mut self) {
|
||||
self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_ilm_expired(&mut self) {
|
||||
self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
fn completed_for_baseline(&self) -> u64 {
|
||||
self.objects_healed
|
||||
.saturating_add(self.objects_failed)
|
||||
.saturating_add(self.skipped_new_versions)
|
||||
.saturating_add(self.skipped_ilm_expired)
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_progress_percentage(&mut self) {
|
||||
if self.objects_total_size > 0 {
|
||||
self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
if self.objects_total_count > 0 {
|
||||
let completed = self.completed_for_baseline();
|
||||
self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
|
||||
let total = self
|
||||
.objects_scanned
|
||||
.saturating_add(self.objects_healed)
|
||||
.saturating_add(self.objects_failed);
|
||||
// calculate progress percentage
|
||||
let total = scanned + healed + failed;
|
||||
if total > 0 {
|
||||
self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
|
||||
self.progress_percentage = (healed as f64 / total as f64) * 100.0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,36 +66,9 @@ impl HealProgress {
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
}
|
||||
|
||||
pub fn refresh_estimated_completion_time(&mut self) {
|
||||
let Some(start_time) = self.start_time else {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
};
|
||||
if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
|
||||
let elapsed = match SystemTime::now().duration_since(start_time) {
|
||||
Ok(elapsed) if !elapsed.is_zero() => elapsed,
|
||||
_ => {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage;
|
||||
self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs));
|
||||
}
|
||||
|
||||
pub fn is_completed(&self) -> bool {
|
||||
if self.progress_percentage >= 100.0 {
|
||||
return true;
|
||||
}
|
||||
if self.objects_total_count > 0 || self.objects_total_size > 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
|
||||
self.progress_percentage >= 100.0
|
||||
|| self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned
|
||||
}
|
||||
|
||||
pub fn get_success_rate(&self) -> f64 {
|
||||
@@ -239,10 +158,6 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 0);
|
||||
assert_eq!(progress.objects_healed, 0);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.skipped_new_versions, 0);
|
||||
assert_eq!(progress.skipped_ilm_expired, 0);
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
assert_eq!(progress.bytes_processed, 0);
|
||||
assert_eq!(progress.progress_percentage, 0.0);
|
||||
assert!(progress.start_time.is_some());
|
||||
@@ -266,73 +181,6 @@ mod tests {
|
||||
assert!(progress.last_update_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_estimates_completion_time_from_progress() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
let eta = progress
|
||||
.estimated_completion_time
|
||||
.expect("partial byte progress should estimate completion");
|
||||
assert!(eta > SystemTime::now());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_byte_baseline_for_percentage() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_object_baseline_when_bytes_unknown() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_counts_skipped_versions_for_object_baseline() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
progress.record_skipped_new_version();
|
||||
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert!((progress.progress_percentage - 60.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_does_not_estimate_completion_without_bytes() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 0);
|
||||
|
||||
assert!(progress.estimated_completion_time.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(1, 1, 0, 1024);
|
||||
|
||||
assert!(!progress.is_completed());
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_update_progress_zero_total() {
|
||||
let mut progress = HealProgress::new();
|
||||
@@ -403,8 +251,6 @@ mod tests {
|
||||
assert_eq!(json["objectsScanned"], 10);
|
||||
assert_eq!(json["objectsHealed"], 8);
|
||||
assert_eq!(json["objectsFailed"], 2);
|
||||
assert_eq!(json["skippedNewVersions"], 0);
|
||||
assert_eq!(json["skippedIlmExpired"], 0);
|
||||
assert_eq!(json["bytesProcessed"], 1024);
|
||||
assert_eq!(json["currentObject"], "test-bucket/test-object");
|
||||
assert!(json["progressPercentage"].is_number());
|
||||
|
||||
@@ -22,7 +22,6 @@ use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
|
||||
use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
@@ -30,37 +29,6 @@ use super::storage_api::storage::{
|
||||
use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct HealBucketUsageBaseline {
|
||||
pub objects_count: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
inner: HealLifecycleExpiryContextInner,
|
||||
}
|
||||
|
||||
enum HealLifecycleExpiryContextInner {
|
||||
Ecstore(EcstoreHealLifecycleExpiryContext),
|
||||
#[allow(dead_code)]
|
||||
Test,
|
||||
}
|
||||
|
||||
impl HealLifecycleExpiryContext {
|
||||
fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Ecstore(inner),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test() -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Test,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
@@ -304,10 +272,6 @@ pub struct HealListItem {
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<HealObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -365,28 +329,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
/// Aggregate usage-cache baselines for the requested buckets.
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Load per-bucket lifecycle expiry context for heal skips.
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Queue lifecycle expiry for a version that heal can skip.
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fix bucket metadata
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
|
||||
|
||||
@@ -467,7 +409,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)>;
|
||||
|
||||
/// List versions for healing via a per-erasure-set DISK-WALK union enumerator
|
||||
@@ -486,10 +427,8 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info)
|
||||
.await
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token).await
|
||||
}
|
||||
|
||||
/// Get disk for resume functionality.
|
||||
@@ -1082,85 +1021,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if buckets.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await {
|
||||
Ok(info) if info.is_complete_bucket_usage_snapshot() => info,
|
||||
Ok(_) | Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let mut baseline = HealBucketUsageBaseline::default();
|
||||
for bucket in buckets {
|
||||
if let Some(usage) = info.buckets_usage.get(bucket) {
|
||||
baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
|
||||
baseline.bytes = baseline.bytes.saturating_add(usage.size);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(baseline))
|
||||
}
|
||||
|
||||
async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await {
|
||||
Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))),
|
||||
Ok(None) => Ok(None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "load_heal_lifecycle_expiry_context",
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry context load failed"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let context = match &context.inner {
|
||||
HealLifecycleExpiryContextInner::Ecstore(context) => context,
|
||||
HealLifecycleExpiryContextInner::Test => return Ok(false),
|
||||
};
|
||||
match self
|
||||
.ecstore
|
||||
.enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info)
|
||||
.await
|
||||
{
|
||||
Ok(queued) => Ok(queued),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "enqueue_heal_lifecycle_expiry",
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry check failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1576,7 +1436,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
loop {
|
||||
let (page_objects, next_token, is_truncated) = self
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false)
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref())
|
||||
.await?;
|
||||
|
||||
all_objects.extend(page_objects);
|
||||
@@ -1611,7 +1471,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1663,19 +1522,10 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let page_objects: Vec<HealListItem> = list_info
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|mut obj| {
|
||||
obj.version_id = obj.version_id.filter(|u| !u.is_nil());
|
||||
let version_id = obj.version_id.map(|u| u.to_string());
|
||||
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
|
||||
let is_delete_marker = obj.delete_marker;
|
||||
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info,
|
||||
is_delete_marker,
|
||||
}
|
||||
.map(|obj| HealListItem {
|
||||
name: obj.name,
|
||||
version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
is_delete_marker: obj.delete_marker,
|
||||
})
|
||||
.collect();
|
||||
let page_count = page_objects.len();
|
||||
@@ -1712,7 +1562,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
// Per-page bounds for the disk-walk union enumerator. Objects are atomic
|
||||
// (never split across pages), so version_budget only bounds how many
|
||||
@@ -1741,16 +1590,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
let (versions, next_forward, is_truncated) = self
|
||||
.ecstore
|
||||
.heal_walk_versions_page(
|
||||
pool_idx,
|
||||
set_idx,
|
||||
bucket,
|
||||
prefix,
|
||||
forward_to.as_deref(),
|
||||
BATCH_OBJECTS,
|
||||
VERSION_BUDGET,
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -1774,8 +1614,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map(|v| HealListItem {
|
||||
name: v.name,
|
||||
version_id: v.version_id,
|
||||
mod_time_unix_nanos: v.mod_time_unix_nanos,
|
||||
lifecycle_object_info: v.lifecycle_object_info,
|
||||
is_delete_marker: v.is_delete_marker,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -12,10 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME,
|
||||
load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
@@ -28,9 +25,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
@@ -39,8 +34,8 @@ pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
|
||||
ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,12 +19,11 @@ use crate::heal::{
|
||||
resume::{
|
||||
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
|
||||
},
|
||||
storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token},
|
||||
storage::{HealStorageAPI, next_heal_listing_token},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -32,7 +31,7 @@ use std::{
|
||||
future::Future,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, Instant, SystemTime},
|
||||
};
|
||||
@@ -179,17 +178,6 @@ pub enum HealPriority {
|
||||
Urgent = 3,
|
||||
}
|
||||
|
||||
impl HealPriority {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Low => "low",
|
||||
Self::Normal => "normal",
|
||||
Self::High => "high",
|
||||
Self::Urgent => "urgent",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealOptions {
|
||||
@@ -351,20 +339,6 @@ impl HealRequest {
|
||||
}
|
||||
|
||||
/// Heal task
|
||||
/// Incremental view over a task's retained result items (HS-06).
|
||||
///
|
||||
/// `next_seq` is the cursor a client should pass on its next poll; `min_seq`
|
||||
/// is the oldest sequence still retained; `lagged` means the client's cursor
|
||||
/// fell behind `min_seq` and items were skipped — the client should restart
|
||||
/// from `min_seq`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealResultWindow {
|
||||
pub items: Vec<HealResultItem>,
|
||||
pub next_seq: u64,
|
||||
pub min_seq: u64,
|
||||
pub lagged: bool,
|
||||
}
|
||||
|
||||
pub struct HealTask {
|
||||
/// Task ID
|
||||
pub id: String,
|
||||
@@ -387,16 +361,8 @@ pub struct HealTask {
|
||||
pub status: Arc<RwLock<HealTaskStatus>>,
|
||||
/// Progress tracking
|
||||
pub progress: Arc<RwLock<HealProgress>>,
|
||||
/// Result items collected from storage heal calls, each stamped with a
|
||||
/// monotonically increasing sequence number for incremental consumption
|
||||
/// (the client passes the last seen seq back and receives only newer
|
||||
/// items; see `get_result_items_since`).
|
||||
pub result_items: Arc<RwLock<Vec<(u64, HealResultItem)>>>,
|
||||
/// Next sequence number to assign; starts at 1.
|
||||
next_item_seq: Arc<AtomicU64>,
|
||||
/// Sequence number of the oldest item still inside the retention window;
|
||||
/// equals `next_item_seq` while the window is empty.
|
||||
min_available_seq: Arc<AtomicU64>,
|
||||
/// Result items collected from storage heal calls.
|
||||
pub result_items: Arc<RwLock<Vec<HealResultItem>>>,
|
||||
result_items_truncated: Arc<AtomicBool>,
|
||||
batch_failure: Arc<RwLock<Option<BatchHealFailure>>>,
|
||||
batch_failure_recorded: Arc<AtomicBool>,
|
||||
@@ -448,8 +414,6 @@ impl HealTask {
|
||||
status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
|
||||
progress: Arc::new(RwLock::new(HealProgress::new())),
|
||||
result_items: Arc::new(RwLock::new(Vec::new())),
|
||||
next_item_seq: Arc::new(AtomicU64::new(1)),
|
||||
min_available_seq: Arc::new(AtomicU64::new(1)),
|
||||
result_items_truncated: Arc::new(AtomicBool::new(false)),
|
||||
batch_failure: Arc::new(RwLock::new(None)),
|
||||
batch_failure_recorded: Arc::new(AtomicBool::new(false)),
|
||||
@@ -534,61 +498,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) {
|
||||
trace_emit(|| {
|
||||
let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
.with_duration(duration)
|
||||
.with_attr("task_id", self.id.as_str())
|
||||
.with_attr("heal_type", self.heal_type.log_kind())
|
||||
.with_attr("state", state)
|
||||
.with_attr("source", self.source.as_str())
|
||||
.with_attr("priority", self.priority.as_str())
|
||||
.with_attr("retry_attempts", u64::from(self.retry_attempts))
|
||||
.with_attr("dry_run", self.options.dry_run);
|
||||
|
||||
event = match &self.heal_type {
|
||||
HealType::Cluster => event,
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()),
|
||||
HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()),
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX);
|
||||
event
|
||||
.with_attr("set_disk_id", set_disk_id.as_str())
|
||||
.with_attr("bucket_count", bucket_count)
|
||||
}
|
||||
HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()),
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::MRF { meta_path } => event.with_object(meta_path.as_str()),
|
||||
};
|
||||
|
||||
match error {
|
||||
Some(error) => event.with_attr("error", error.to_string()),
|
||||
None => event,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn remaining_timeout(&self) -> Result<Option<Duration>> {
|
||||
if let Some(total) = self.options.timeout {
|
||||
let start_instant = { *self.task_start_instant.read().await };
|
||||
@@ -808,7 +717,6 @@ impl HealTask {
|
||||
queue_delay = ?queue_delay,
|
||||
"Heal task started"
|
||||
});
|
||||
self.emit_trace_task_state("started", Duration::ZERO, None);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
@@ -897,14 +805,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
let terminal_state = match &result {
|
||||
Ok(_) => "completed",
|
||||
Err(Error::TaskCancelled) => "cancelled",
|
||||
Err(Error::TaskTimeout) => "timed_out",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err());
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
@@ -935,63 +835,18 @@ impl HealTask {
|
||||
}
|
||||
|
||||
pub async fn get_result_items(&self) -> Vec<HealResultItem> {
|
||||
self.result_items.read().await.iter().map(|(_, item)| item.clone()).collect()
|
||||
}
|
||||
|
||||
/// Sequence-stamped retained window, used when archiving a completed
|
||||
/// task so incremental cursors survive the transition (HS-06).
|
||||
pub async fn get_seqed_result_items(&self) -> Vec<(u64, HealResultItem)> {
|
||||
self.result_items.read().await.clone()
|
||||
}
|
||||
|
||||
/// Incremental result window (HS-06): `since = None` returns the full
|
||||
/// retained window (legacy snapshot semantics); `since = Some(seq)`
|
||||
/// returns only items stamped with a sequence greater than `seq`.
|
||||
/// `lagged` warns that the caller's cursor fell behind the window start
|
||||
/// and items were skipped (the response carries `min_seq` as the catch-up
|
||||
/// cursor).
|
||||
pub async fn get_result_items_since(&self, since: Option<u64>) -> HealResultWindow {
|
||||
let result_items = self.result_items.read().await;
|
||||
let next_seq = self.next_item_seq.load(Ordering::Relaxed);
|
||||
let min_seq = self.min_available_seq.load(Ordering::Relaxed);
|
||||
let mut lagged = false;
|
||||
let items = match since {
|
||||
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
|
||||
Some(cursor) => {
|
||||
if cursor + 1 < min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
result_items
|
||||
.iter()
|
||||
.filter(|(seq, _)| *seq > cursor)
|
||||
.map(|(_, item)| item.clone())
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
};
|
||||
HealResultWindow {
|
||||
items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
lagged,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn result_items_truncated(&self) -> bool {
|
||||
self.result_items_truncated.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
async fn record_result_item(&self, result: HealResultItem) {
|
||||
let seq = self.next_item_seq.fetch_add(1, Ordering::Relaxed);
|
||||
let mut result_items = self.result_items.write().await;
|
||||
if result_items.len() < MAX_RETAINED_HEAL_RESULT_ITEMS {
|
||||
result_items.push((seq, result));
|
||||
result_items.push(result);
|
||||
} else {
|
||||
// Slide the window: the oldest item leaves and the cursor for the
|
||||
// oldest still-available item moves forward with it.
|
||||
result_items.remove(0);
|
||||
self.min_available_seq
|
||||
.store(result_items.first().map_or(seq, |(oldest, _)| *oldest), Ordering::Relaxed);
|
||||
result_items.push((seq, result));
|
||||
self.result_items_truncated.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -1680,7 +1535,7 @@ impl HealTask {
|
||||
let (objects, next_token, is_truncated) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1842,23 +1697,6 @@ impl HealTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
|
||||
let baseline = match self
|
||||
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
|
||||
.await
|
||||
{
|
||||
Ok(Some(baseline)) => baseline,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total_baseline(objects_count, bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -2460,8 +2298,6 @@ impl HealTask {
|
||||
None
|
||||
};
|
||||
|
||||
self.apply_erasure_set_usage_baseline(&buckets).await?;
|
||||
|
||||
let healing_marker = format!("{set_disk_id}:{}", self.id);
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
let state = resume_manager.get_state().await;
|
||||
@@ -2766,8 +2602,7 @@ impl HealTask {
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
let bytes_processed = progress.bytes_processed;
|
||||
progress.update_progress(4, 4, 0, bytes_processed);
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
|
||||
match result {
|
||||
@@ -2823,7 +2658,6 @@ mod tests {
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk};
|
||||
use super::*;
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
@@ -3369,8 +3203,6 @@ mod tests {
|
||||
block_heal_object: Mutex<bool>,
|
||||
resume_disk: Mutex<Option<DiskStore>>,
|
||||
replacement_resume_disk: Mutex<Option<DiskStore>>,
|
||||
usage_baseline: Mutex<Option<HealBucketUsageBaseline>>,
|
||||
usage_baseline_error: Mutex<bool>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3433,69 +3265,11 @@ mod tests {
|
||||
assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_emits_heal_trace_task_state() {
|
||||
let mut trace = subscribe_trace_events();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let started = recv_trace_task_state(&mut trace, &task.id, "started").await;
|
||||
assert_eq!(started.kind, TraceKind::Heal);
|
||||
assert_eq!(started.func, TraceFunc::HealTask);
|
||||
assert_eq!(started.bucket.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(started.object.as_deref(), Some("object-a"));
|
||||
assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object"));
|
||||
assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal"));
|
||||
assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a"));
|
||||
|
||||
let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await;
|
||||
assert_eq!(completed.kind, TraceKind::Heal);
|
||||
assert_eq!(completed.func, TraceFunc::HealTask);
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if trace_attr_string(&event, "task_id").as_deref() == Some(task_id)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected trace state {state} for task {task_id}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
TraceVal::Bool(value) => value.to_string(),
|
||||
TraceVal::U64(value) => value.to_string(),
|
||||
TraceVal::I64(value) => value.to_string(),
|
||||
TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a latest, non-delete-marker heal list item with no version id.
|
||||
fn heal_item(name: &str) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: None,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
@@ -3583,13 +3357,6 @@ mod tests {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if *self.usage_baseline_error.lock().unwrap() {
|
||||
return Err(Error::Other("usage baseline unavailable".to_string()));
|
||||
}
|
||||
Ok(*self.usage_baseline.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -3773,7 +3540,6 @@ mod tests {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
|
||||
if *self.truncate_without_token.lock().unwrap() {
|
||||
@@ -3949,69 +3715,6 @@ mod tests {
|
||||
assert!(task.result_items_truncated());
|
||||
}
|
||||
|
||||
// HS-06 (backlog#1870): incremental result windows.
|
||||
#[tokio::test]
|
||||
async fn result_items_seq_is_monotonic_and_incremental_slices_work() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket-a".to_string()), storage);
|
||||
|
||||
for round in 0..5u64 {
|
||||
let item = HealResultItem {
|
||||
object_size: round as usize,
|
||||
..Default::default()
|
||||
};
|
||||
task.record_result_item(item).await;
|
||||
}
|
||||
|
||||
let full = task.get_result_items_since(None).await;
|
||||
assert_eq!(full.items.len(), 5, "None keeps the full-snapshot semantics");
|
||||
assert_eq!(full.next_seq, 6, "next_seq is one past the last assigned");
|
||||
assert_eq!(full.min_seq, 1, "nothing was evicted yet");
|
||||
assert!(!full.lagged);
|
||||
|
||||
// Incremental: only items newer than the cursor.
|
||||
let incremental = task.get_result_items_since(Some(3)).await;
|
||||
assert_eq!(
|
||||
incremental.items.iter().map(|item| item.object_size).collect::<Vec<_>>(),
|
||||
vec![3, 4],
|
||||
"only sequences greater than the cursor are returned"
|
||||
);
|
||||
assert_eq!(incremental.next_seq, 6);
|
||||
|
||||
// A cursor at the head is not lagging.
|
||||
assert!(!task.get_result_items_since(Some(0)).await.lagged);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn result_items_window_slide_moves_min_seq_and_flags_lagging_cursors() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket-a".to_string()), storage);
|
||||
|
||||
// Fill the window completely, then push two more items: seq 1 and 2
|
||||
// are evicted by the slide.
|
||||
for _ in 0..(MAX_RETAINED_HEAL_RESULT_ITEMS + 2) {
|
||||
task.record_result_item(HealResultItem::default()).await;
|
||||
}
|
||||
|
||||
let full = task.get_result_items_since(None).await;
|
||||
assert_eq!(full.items.len(), MAX_RETAINED_HEAL_RESULT_ITEMS);
|
||||
assert_eq!(full.min_seq, 3, "each evicted head item moved the oldest-available cursor");
|
||||
assert!(task.result_items_truncated());
|
||||
|
||||
// A client still polling from before the eviction is lagging.
|
||||
let lagging = task.get_result_items_since(Some(0)).await;
|
||||
assert!(lagging.lagged, "a cursor behind min_seq must be flagged");
|
||||
assert_eq!(lagging.min_seq, 3, "the response tells the client where to restart");
|
||||
|
||||
// A cursor inside the window is fine.
|
||||
assert!(!task.get_result_items_since(Some(3)).await.lagged);
|
||||
|
||||
// The lagging client restarts from min_seq and gets the full window.
|
||||
let catch_up = task.get_result_items_since(Some(3)).await;
|
||||
assert_eq!(catch_up.items.len(), MAX_RETAINED_HEAL_RESULT_ITEMS - 1);
|
||||
assert!(!catch_up.lagged);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -4951,73 +4654,6 @@ mod tests {
|
||||
assert!(storage.object_heal_opts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_applies_usage_baseline_to_progress() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
|
||||
objects_count: 10,
|
||||
bytes: 8,
|
||||
})),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("erasure set heal should complete");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 10);
|
||||
assert_eq!(progress.objects_total_size, 8);
|
||||
assert_eq!(progress.bytes_processed, 2);
|
||||
assert!((progress.progress_percentage - 25.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_ignores_usage_baseline_errors() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline_error: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("usage baseline failures should not fail erasure set heal");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
|
||||
@@ -158,10 +158,6 @@ pub async fn init_heal_manager_with_workload_provider(
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Start the MRF intent consumer (error-path repair intents + durable
|
||||
// journal replay) now that the manager can accept submissions.
|
||||
heal::mrf_queue::spawn_mrf_consumer(heal_manager.clone());
|
||||
|
||||
#[cfg(test)]
|
||||
test_hook_after_manager_start().await;
|
||||
|
||||
@@ -449,7 +445,6 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool), Error> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc<ECStoreHealStorage>, bucket:
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.await
|
||||
.expect("list_objects_for_heal_page failed");
|
||||
items.extend(page);
|
||||
|
||||
@@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc<ECStoreHealStorage>, bucket: &str) -> V
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.await
|
||||
.expect("b5 list page failed");
|
||||
items.extend(page);
|
||||
@@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc<ECStoreHealStorage>, bucket: &st
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false)
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref())
|
||||
.await
|
||||
.expect("disk-walk list page failed");
|
||||
items.extend(page);
|
||||
@@ -418,7 +418,7 @@ mod serial_tests {
|
||||
let mut pages = 0usize;
|
||||
loop {
|
||||
let (versions, next_forward, truncated) = ecstore
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false)
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000)
|
||||
.await
|
||||
.expect("heal_walk_versions_page failed");
|
||||
pages += 1;
|
||||
|
||||
@@ -242,7 +242,6 @@ fn test_heal_task_status_atomic_update() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
@@ -386,7 +385,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! HS-01 (rustfs/backlog#1865): MRF intent pipeline integration tests.
|
||||
//!
|
||||
//! Drives the real consumer loop (`spawn_mrf_consumer`) against a real
|
||||
//! 4-disk `ECStore` heal storage and a `HealManager` that has not started its
|
||||
//! scheduler, so submitted intents stay observable in the admission queue.
|
||||
//! Under `cargo nextest` each test runs in its own process, which keeps the
|
||||
//! process-global MRF channel singleton safe.
|
||||
|
||||
use rustfs_common::mrf_channel::{self, MrfKind};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
mrf_queue,
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
|
||||
const META_BUCKET: &str = ".rustfs.sys";
|
||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_mrf_test")
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, heal_storage)
|
||||
}
|
||||
|
||||
fn make_manager(storage: Arc<dyn HealStorageAPI>) -> Arc<HealManager> {
|
||||
Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
// Keep the scheduler from draining the queue before assertions.
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
/// Encode one journal record independently of the implementation, so a format
|
||||
/// drift between writer and this fixture fails loudly here.
|
||||
fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]>, attempts: u8) -> Vec<u8> {
|
||||
let mut body = vec![1u8, 1, kind, attempts];
|
||||
body.extend_from_slice(&1_700_000_000_000u64.to_le_bytes());
|
||||
match version {
|
||||
Some(bytes) => {
|
||||
body.push(1);
|
||||
body.extend_from_slice(&bytes);
|
||||
}
|
||||
None => body.push(0),
|
||||
}
|
||||
body.extend_from_slice(&(bucket.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(&(object.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(bucket.as_bytes());
|
||||
body.extend_from_slice(object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&body);
|
||||
body.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
|
||||
body
|
||||
}
|
||||
|
||||
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
for path in disk_paths {
|
||||
let journal = path.join(META_BUCKET).join(JOURNAL_REL);
|
||||
std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
|
||||
std::fs::write(&journal, data).expect("write journal fixture");
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: std::future::Future<Output = bool>,
|
||||
{
|
||||
let start = std::time::Instant::now();
|
||||
while start.elapsed() < deadline {
|
||||
if probe().await {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// A decode-failure intent delivered on the global channel must surface in the
|
||||
/// heal manager as an Urgent request attributed to the MRF source.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
||||
let (_disk_paths, storage) = heal_env().await;
|
||||
let manager = make_manager(storage);
|
||||
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
|
||||
assert!(
|
||||
mrf_channel::try_send_mrf_intent(MrfKind::DecodeFailure, "mrf-bucket", "mrf-object", None),
|
||||
"intent should be accepted while the consumer holds the channel"
|
||||
);
|
||||
|
||||
let appeared = wait_until(Duration::from_secs(10), || async {
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
snapshot.queued_by_source.mrf >= 1 && snapshot.queued_by_priority.urgent >= 1
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
appeared,
|
||||
"MRF intent must reach the manager queue as an Urgent request (snapshot: {:?})",
|
||||
manager.operations_snapshot().await
|
||||
);
|
||||
}
|
||||
|
||||
/// A journal left behind by a previous process must be replayed into the
|
||||
/// manager queue and then removed, and a torn tail must not block replay of
|
||||
/// the intact records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
|
||||
// The journal reader resolves disks through the process-local disk map;
|
||||
// register the environment's disks the same way server startup does.
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "mrf-test".to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
|
||||
let mut journal = journal_record(1, "replay-bucket", "replay-object", Some([9u8; 16]), 0);
|
||||
journal.extend(journal_record(3, "replay-bucket", "partial-object", None, 1));
|
||||
// Torn tail: a third record truncated mid-way must not block the two
|
||||
// intact records above.
|
||||
journal.extend_from_slice(&journal_record(2, "replay-bucket", "metadata-object", None, 0)[..8]);
|
||||
write_journal_to_disks(&disk_paths, &journal);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
// Replay directly (not via the process-global channel consumer, which the
|
||||
// sibling test already claimed in this process under plain `cargo test`).
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 2, "the two intact records must be replayed");
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 2, "replayed intents must be attributed to the MRF source");
|
||||
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"the journal file must be removed after a successful replay"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
|
||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||
}
|
||||
@@ -9,9 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Removed
|
||||
|
||||
#### rustfs-io-core
|
||||
- **Zero-consumer modules** (added in 0.0.5): `reader`, `writer`, `bufreader_optimizer`, `shared_memory`, `direct_io`, `timeout_wrapper`, `io_priority_queue`, and `scheduler` had no caller in the workspace and were removed (rustfs/backlog#1824). The scheduling algorithm and the request timeout wrapper that RustFS actually runs live in `rustfs/src/storage/`; this crate keeps the config shapes they project into. `OperationProgress` moved to the new `progress` module and is still exported as `rustfs_io_core::OperationProgress`.
|
||||
|
||||
#### rustfs-io-metrics
|
||||
- **Unified configuration** (added in 0.0.5): the zero-consumer `IoConfig`, `CacheSettings`, `IoSchedulerSettings`, `BackpressureSettings`, `TimeoutSettings`, `DeadlockDetectionSettings` types and their `DEFAULT_*` constants were removed (rustfs/rustfs#6008); rustfs-io-core's `IoSchedulerConfig`/`BackpressureConfig` remain the canonical configuration types.
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "Shared I/O primitives for RustFS (buffer pool, storage profiling, backpressure, deadlock detection)"
|
||||
keywords = ["io", "buffer", "pool", "rustfs", "backpressure"]
|
||||
description = "Buffered I/O reader and writer implementations for RustFS (mmap-then-copy, aligned pread)"
|
||||
keywords = ["io", "reader", "writer", "rustfs", "mmap"]
|
||||
categories = ["development-tools", "filesystem"]
|
||||
|
||||
[lints]
|
||||
@@ -38,6 +38,7 @@ hotpath.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
|
||||
memmap2 = { workspace = true }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
|
||||
+120
-18
@@ -23,20 +23,67 @@
|
||||
|
||||
## Overview
|
||||
|
||||
**rustfs-io-core** holds the shared I/O primitives for [RustFS](https://rustfs.com), a distributed object storage system. It provides:
|
||||
**rustfs-io-core** is the core I/O scheduling module for [RustFS](https://rustfs.com), a distributed object storage system. It provides:
|
||||
|
||||
- **Buffer Pool**: Tiered `BytesPool` for buffer reuse
|
||||
- **Storage Profiling**: Storage-media and access-pattern model (`io_profile`)
|
||||
- **Scheduler Configuration**: The `IoSchedulerConfig` / `IoPriorityQueueConfig` shapes the storage layer projects into
|
||||
- **I/O Scheduler**: Adaptive buffer size calculation and load management
|
||||
- **Priority Queue**: Request priority scheduling with starvation prevention
|
||||
- **Backpressure Control**: System overload protection with graceful degradation
|
||||
- **Deadlock Detection**: Wait-for graph based deadlock detection algorithm
|
||||
- **Lock Optimizer**: Adaptive spin lock optimization
|
||||
- **Progress Tracking**: Byte progress and staleness for long-running operations
|
||||
|
||||
The scheduling algorithm itself lives in `rustfs/src/storage/concurrency/io_schedule.rs`; this crate carries the configuration shapes it projects into, not a second implementation.
|
||||
- **Timeout Wrapper**: Dynamic timeout calculation and operation progress tracking
|
||||
|
||||
## Features
|
||||
|
||||
### I/O Scheduler
|
||||
|
||||
Adaptive I/O scheduling with dynamic buffer size calculation based on file size, access pattern, and system load:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{IoScheduler, IoSchedulerConfig, IoLoadLevel};
|
||||
use rustfs_io_core::io_profile::{StorageMedia, AccessPattern};
|
||||
|
||||
// Create scheduler
|
||||
let config = IoSchedulerConfig {
|
||||
max_concurrent_reads: 64,
|
||||
base_buffer_size: 64 * 1024, // 64 KB
|
||||
max_buffer_size: 1024 * 1024, // 1 MB
|
||||
..Default::default()
|
||||
};
|
||||
let scheduler = IoScheduler::new(config);
|
||||
|
||||
// Calculate optimal buffer size
|
||||
let buffer_size = calculate_optimal_buffer_size(
|
||||
10 * 1024 * 1024, // 10 MB file
|
||||
64 * 1024, // base buffer
|
||||
true, // sequential access
|
||||
4, // concurrent requests
|
||||
StorageMedia::Ssd,
|
||||
IoLoadLevel::Low,
|
||||
);
|
||||
```
|
||||
|
||||
### Priority Queue
|
||||
|
||||
Priority queue with starvation prevention:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{IoPriorityQueue, IoPriority, IoQueueStatus};
|
||||
|
||||
let queue = IoPriorityQueue::<()>::new(100);
|
||||
|
||||
// Enqueue request
|
||||
let request_id = queue.enqueue(IoPriority::High, (), 1024);
|
||||
|
||||
// Dequeue request
|
||||
if let Some((priority, data)) = queue.dequeue() {
|
||||
println!("Processing priority {:?} request", priority);
|
||||
}
|
||||
|
||||
// Check queue status
|
||||
let status = queue.status();
|
||||
println!("High priority waiting: {}", status.high_priority_waiting);
|
||||
```
|
||||
|
||||
### Backpressure Control
|
||||
|
||||
System overload protection:
|
||||
@@ -101,23 +148,71 @@ let stats = optimizer.stats();
|
||||
println!("Locks acquired: {}", stats.total_acquired());
|
||||
```
|
||||
|
||||
### Progress Tracking
|
||||
### Timeout Wrapper
|
||||
|
||||
Byte progress and staleness for long-running operations:
|
||||
Dynamic timeout calculation:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::OperationProgress;
|
||||
use rustfs_io_core::{RequestTimeoutWrapper, TimeoutConfig};
|
||||
use std::time::Duration;
|
||||
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
let config = TimeoutConfig {
|
||||
base_timeout: Duration::from_secs(5),
|
||||
timeout_per_mb: Duration::from_millis(100),
|
||||
max_timeout: Duration::from_secs(300),
|
||||
..Default::default()
|
||||
};
|
||||
let wrapper = RequestTimeoutWrapper::new(config);
|
||||
|
||||
progress.update(500);
|
||||
assert_eq!(progress.progress_percent(), Some(50.0));
|
||||
assert!(!progress.is_stale());
|
||||
// Calculate operation timeout
|
||||
let timeout = wrapper.calculate_timeout(10 * 1024 * 1024); // 10 MB
|
||||
```
|
||||
|
||||
## Buffer Size Calculation
|
||||
|
||||
Multiple buffer size calculation functions are provided:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{
|
||||
get_concurrency_aware_buffer_size,
|
||||
get_advanced_buffer_size,
|
||||
get_buffer_size_for_media,
|
||||
calculate_optimal_buffer_size,
|
||||
KI_B, MI_B,
|
||||
};
|
||||
use rustfs_io_core::io_profile::StorageMedia;
|
||||
|
||||
// Basic calculation
|
||||
let size1 = get_concurrency_aware_buffer_size(1024 * 1024, 64 * 1024);
|
||||
|
||||
// Advanced calculation (considering access pattern)
|
||||
let size2 = get_advanced_buffer_size(10 * 1024 * 1024, 64 * 1024, true);
|
||||
|
||||
// Media type optimization
|
||||
let size3 = get_buffer_size_for_media(64 * 1024, StorageMedia::Ssd);
|
||||
|
||||
// Comprehensive calculation
|
||||
let size4 = calculate_optimal_buffer_size(
|
||||
100 * 1024 * 1024, // 100 MB file
|
||||
64 * 1024, // base buffer
|
||||
true, // sequential access
|
||||
4, // concurrent requests
|
||||
StorageMedia::Nvme,
|
||||
IoLoadLevel::Low,
|
||||
);
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `RUSTFS_MAX_CONCURRENT_READS` | Max concurrent reads | 64 |
|
||||
| `RUSTFS_BASE_BUFFER_SIZE` | Base buffer size | 65536 |
|
||||
| `RUSTFS_MAX_BUFFER_SIZE` | Max buffer size | 1048576 |
|
||||
| `RUSTFS_IO_TIMEOUT_SECS` | I/O timeout seconds | 30 |
|
||||
|
||||
### Code Configuration
|
||||
|
||||
```rust
|
||||
@@ -145,11 +240,12 @@ rustfs-io-core/
|
||||
├── src/
|
||||
│ ├── lib.rs # Module entry
|
||||
│ ├── config.rs # Configuration types
|
||||
│ ├── pool.rs # Tiered buffer pool
|
||||
│ ├── scheduler.rs # I/O scheduler
|
||||
│ ├── io_priority_queue.rs # Priority queue
|
||||
│ ├── backpressure.rs # Backpressure control
|
||||
│ ├── deadlock_detector.rs # Deadlock detection
|
||||
│ ├── lock_optimizer.rs # Lock optimization
|
||||
│ ├── progress.rs # Operation progress tracking
|
||||
│ ├── timeout_wrapper.rs # Timeout wrapper
|
||||
│ └── io_profile.rs # I/O profile
|
||||
└── Cargo.toml
|
||||
```
|
||||
@@ -158,15 +254,21 @@ rustfs-io-core/
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
cargo nextest run --package rustfs-io-core
|
||||
cargo test --package rustfs-io-core
|
||||
|
||||
# Run specific tests
|
||||
cargo nextest run --package rustfs-io-core -E 'test(backpressure)'
|
||||
cargo test --package rustfs-io-core --lib scheduler
|
||||
|
||||
# Run benchmarks
|
||||
cargo bench --package rustfs-io-core
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- [API Documentation](https://docs.rs/rustfs-io-core)
|
||||
- [I/O Scheduler Design](./docs/scheduler-design.md)
|
||||
- [Backpressure Control Design](./docs/backpressure-design.md)
|
||||
- [Deadlock Detection Algorithm](./docs/deadlock-detection.md)
|
||||
|
||||
## Related Modules
|
||||
|
||||
|
||||
+131
-18
@@ -23,20 +23,71 @@
|
||||
|
||||
## 📖 概述
|
||||
|
||||
**rustfs-io-core** 是 [RustFS](https://rustfs.com) 分布式对象存储系统的共享 I/O 基础组件。它提供了:
|
||||
**rustfs-io-core** 是 [RustFS](https://rustfs.com) 分布式对象存储系统的核心 I/O 调度模块。它提供了:
|
||||
|
||||
- **缓冲池**:分级复用的 `BytesPool`
|
||||
- **存储画像**:存储介质与访问模式模型(`io_profile`)
|
||||
- **调度配置**:存储层投影使用的 `IoSchedulerConfig` / `IoPriorityQueueConfig`
|
||||
- **I/O 调度器**:自适应缓冲区大小计算和负载管理
|
||||
- **优先级队列**:支持饥饿预防的请求优先级调度
|
||||
- **背压控制**:系统过载保护和优雅降级
|
||||
- **死锁检测**:基于等待图的死锁检测算法
|
||||
- **锁优化**:自适应自旋锁优化
|
||||
- **进度追踪**:长耗时操作的字节进度与停滞判定
|
||||
|
||||
调度算法本身位于 `rustfs/src/storage/concurrency/io_schedule.rs`;本 crate 只承载它投影使用的配置形状,不是第二套实现。
|
||||
- **超时包装器**:动态超时计算和操作进度追踪
|
||||
|
||||
## ✨ 核心功能
|
||||
|
||||
### I/O 调度器 (IoScheduler)
|
||||
|
||||
自适应 I/O 调度,根据文件大小、访问模式和系统负载动态调整缓冲区大小:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{IoScheduler, IoSchedulerConfig, IoLoadLevel};
|
||||
use rustfs_io_core::io_profile::{StorageMedia, AccessPattern};
|
||||
|
||||
// 创建调度器
|
||||
let config = IoSchedulerConfig {
|
||||
max_concurrent_reads: 64,
|
||||
base_buffer_size: 64 * 1024, // 64 KB
|
||||
max_buffer_size: 1024 * 1024, // 1 MB
|
||||
..Default::default()
|
||||
};
|
||||
let scheduler = IoScheduler::new(config);
|
||||
|
||||
// 计算最优缓冲区大小
|
||||
let buffer_size = scheduler.calculate_buffer_size(
|
||||
10 * 1024 * 1024, // 10 MB 文件
|
||||
true, // 顺序访问
|
||||
StorageMedia::Ssd,
|
||||
IoLoadLevel::Low,
|
||||
);
|
||||
println!("缓冲区大小: {} bytes", buffer_size);
|
||||
```
|
||||
|
||||
### 优先级队列 (IoPriorityQueue)
|
||||
|
||||
支持饥饿预防的优先级队列:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{IoPriorityQueue, IoPriority, IoQueueStatus};
|
||||
|
||||
let queue = IoPriorityQueue::<()>::new(100);
|
||||
|
||||
// 入队请求
|
||||
let request_id = queue.enqueue(
|
||||
IoPriority::High,
|
||||
(), // 请求数据
|
||||
1024, // 请求大小
|
||||
);
|
||||
|
||||
// 出队请求
|
||||
if let Some((priority, data)) = queue.dequeue() {
|
||||
println!("处理优先级 {:?} 的请求", priority);
|
||||
}
|
||||
|
||||
// 检查队列状态
|
||||
let status = queue.status();
|
||||
println!("高优先级等待: {}", status.high_priority_waiting);
|
||||
println!("低优先级等待: {}", status.low_priority_waiting);
|
||||
```
|
||||
|
||||
### 背压控制 (BackpressureMonitor)
|
||||
|
||||
系统过载保护:
|
||||
@@ -114,23 +165,78 @@ let stats = optimizer.stats();
|
||||
println!("获取锁次数: {}", stats.locks_acquired.load(std::sync::atomic::Ordering::Relaxed));
|
||||
```
|
||||
|
||||
### 进度追踪 (OperationProgress)
|
||||
### 超时包装器 (RequestTimeoutWrapper)
|
||||
|
||||
长耗时操作的字节进度与停滞判定:
|
||||
动态超时计算:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::OperationProgress;
|
||||
use rustfs_io_core::{RequestTimeoutWrapper, TimeoutConfig};
|
||||
use std::time::Duration;
|
||||
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
let config = TimeoutConfig {
|
||||
base_timeout: Duration::from_secs(5),
|
||||
timeout_per_mb: Duration::from_millis(100),
|
||||
max_timeout: Duration::from_secs(300),
|
||||
..Default::default()
|
||||
};
|
||||
let wrapper = RequestTimeoutWrapper::new(config);
|
||||
|
||||
progress.update(500);
|
||||
assert_eq!(progress.progress_percent(), Some(50.0));
|
||||
assert!(!progress.is_stale());
|
||||
// 计算操作超时
|
||||
let timeout = wrapper.calculate_timeout(10 * 1024 * 1024); // 10 MB
|
||||
println!("超时时间: {:?}", timeout);
|
||||
|
||||
// 执行带超时的操作
|
||||
let result = wrapper.execute_with_timeout(async {
|
||||
// 异步操作
|
||||
Ok::<_, std::io::Error>(())
|
||||
}, timeout).await;
|
||||
```
|
||||
|
||||
## 📊 缓冲区大小计算
|
||||
|
||||
模块提供了多种缓冲区大小计算函数:
|
||||
|
||||
```rust
|
||||
use rustfs_io_core::{
|
||||
get_concurrency_aware_buffer_size,
|
||||
get_advanced_buffer_size,
|
||||
get_buffer_size_for_media,
|
||||
calculate_optimal_buffer_size,
|
||||
KI_B, MI_B,
|
||||
};
|
||||
use rustfs_io_core::io_profile::StorageMedia;
|
||||
|
||||
// 基础计算
|
||||
let size1 = get_concurrency_aware_buffer_size(1024 * 1024, 64 * 1024);
|
||||
|
||||
// 高级计算(考虑访问模式)
|
||||
let size2 = get_advanced_buffer_size(10 * 1024 * 1024, 64 * 1024, true);
|
||||
|
||||
// 媒体类型优化
|
||||
let size3 = get_buffer_size_for_media(64 * 1024, StorageMedia::Ssd);
|
||||
|
||||
// 综合计算
|
||||
let size4 = calculate_optimal_buffer_size(
|
||||
100 * 1024 * 1024, // 100 MB 文件
|
||||
64 * 1024, // 基础缓冲区
|
||||
true, // 顺序访问
|
||||
4, // 并发请求数
|
||||
StorageMedia::Nvme,
|
||||
IoLoadLevel::Low,
|
||||
);
|
||||
```
|
||||
|
||||
## 🔧 配置
|
||||
|
||||
### 环境变量
|
||||
|
||||
| 变量名 | 描述 | 默认值 |
|
||||
|--------|------|--------|
|
||||
| `RUSTFS_MAX_CONCURRENT_READS` | 最大并发读数 | 64 |
|
||||
| `RUSTFS_BASE_BUFFER_SIZE` | 基础缓冲区大小 | 65536 |
|
||||
| `RUSTFS_MAX_BUFFER_SIZE` | 最大缓冲区大小 | 1048576 |
|
||||
| `RUSTFS_IO_TIMEOUT_SECS` | I/O 超时秒数 | 30 |
|
||||
|
||||
### 代码配置
|
||||
|
||||
```rust
|
||||
@@ -158,11 +264,12 @@ rustfs-io-core/
|
||||
├── src/
|
||||
│ ├── lib.rs # 模块入口
|
||||
│ ├── config.rs # 配置类型
|
||||
│ ├── pool.rs # 分级缓冲池
|
||||
│ ├── scheduler.rs # I/O 调度器
|
||||
│ ├── io_priority_queue.rs # 优先级队列
|
||||
│ ├── backpressure.rs # 背压控制
|
||||
│ ├── deadlock_detector.rs # 死锁检测
|
||||
│ ├── lock_optimizer.rs # 锁优化
|
||||
│ ├── progress.rs # 操作进度追踪
|
||||
│ ├── timeout_wrapper.rs # 超时包装器
|
||||
│ └── io_profile.rs # I/O 配置文件
|
||||
└── Cargo.toml
|
||||
```
|
||||
@@ -171,15 +278,21 @@ rustfs-io-core/
|
||||
|
||||
```bash
|
||||
# 运行所有测试
|
||||
cargo nextest run --package rustfs-io-core
|
||||
cargo test --package rustfs-io-core
|
||||
|
||||
# 运行特定测试
|
||||
cargo nextest run --package rustfs-io-core -E 'test(backpressure)'
|
||||
cargo test --package rustfs-io-core --lib scheduler
|
||||
|
||||
# 运行基准测试
|
||||
cargo bench --package rustfs-io-core
|
||||
```
|
||||
|
||||
## 📚 文档
|
||||
|
||||
- [API 文档](https://docs.rs/rustfs-io-core)
|
||||
- [I/O 调度器设计](./docs/scheduler-design.md)
|
||||
- [背压控制原理](./docs/backpressure-design.md)
|
||||
- [死锁检测算法](./docs/deadlock-detection.md)
|
||||
|
||||
## 🔗 相关模块
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Example demonstrating I/O scheduler usage.
|
||||
|
||||
use rustfs_io_core::io_profile::StorageMedia;
|
||||
use rustfs_io_core::{
|
||||
BackpressureMonitor, BackpressureState, DeadlockDetector, IoLoadLevel, IoScheduler, IoSchedulerConfig, KI_B, LockOptimizer,
|
||||
LockType, MI_B, calculate_optimal_buffer_size, get_buffer_size_for_media,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
fn main() {
|
||||
println!("=== rustfs-io-core Example ===\n");
|
||||
|
||||
// 1. I/O scheduler example
|
||||
io_scheduler_example();
|
||||
|
||||
// 2. Buffer size calculation example
|
||||
buffer_size_example();
|
||||
|
||||
// 3. Backpressure control example
|
||||
backpressure_example();
|
||||
|
||||
// 4. Deadlock detection example
|
||||
deadlock_detection_example();
|
||||
|
||||
// 5. Lock optimizer example
|
||||
lock_optimizer_example();
|
||||
}
|
||||
|
||||
fn io_scheduler_example() {
|
||||
println!("--- I/O Scheduler ---");
|
||||
|
||||
// Create scheduler with configuration
|
||||
let config = IoSchedulerConfig {
|
||||
max_concurrent_reads: 64,
|
||||
base_buffer_size: 64 * KI_B,
|
||||
max_buffer_size: MI_B,
|
||||
..Default::default()
|
||||
};
|
||||
let scheduler = IoScheduler::new(config);
|
||||
|
||||
println!(" Max concurrent reads: {}", scheduler.config().max_concurrent_reads);
|
||||
println!(" Base buffer size: {} KB", scheduler.config().base_buffer_size / KI_B);
|
||||
println!(" Max buffer size: {} KB", scheduler.config().max_buffer_size / KI_B);
|
||||
|
||||
// Calculate buffer sizes for different scenarios
|
||||
let scenarios = [
|
||||
("Small file", 10 * KI_B as i64, true, StorageMedia::Ssd),
|
||||
("Medium file", MI_B as i64, true, StorageMedia::Ssd),
|
||||
("Large sequential", 100 * MI_B as i64, true, StorageMedia::Ssd),
|
||||
("Large random", 100 * MI_B as i64, false, StorageMedia::Ssd),
|
||||
("NVMe large", 100 * MI_B as i64, true, StorageMedia::Nvme),
|
||||
("HDD large", 100 * MI_B as i64, true, StorageMedia::Hdd),
|
||||
];
|
||||
|
||||
for (name, size, sequential, media) in scenarios {
|
||||
let buffer = calculate_optimal_buffer_size(size, 64 * KI_B, sequential, 4, media, IoLoadLevel::Low);
|
||||
println!(" {}: {} bytes ({} KB)", name, buffer, buffer / KI_B);
|
||||
}
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
fn buffer_size_example() {
|
||||
println!("--- Buffer Size Calculation ---");
|
||||
|
||||
// Comprehensive calculation
|
||||
let size1 = calculate_optimal_buffer_size(10 * MI_B as i64, 64 * KI_B, true, 4, StorageMedia::Ssd, IoLoadLevel::Low);
|
||||
println!(" Comprehensive (10MB, sequential, SSD): {} KB", size1 / KI_B);
|
||||
|
||||
// Media type optimization
|
||||
let media_types = [
|
||||
StorageMedia::Nvme,
|
||||
StorageMedia::Ssd,
|
||||
StorageMedia::Hdd,
|
||||
StorageMedia::Unknown,
|
||||
];
|
||||
for media in media_types {
|
||||
let size = get_buffer_size_for_media(64 * KI_B, media);
|
||||
println!(" {} optimized: {} KB", media.as_str(), size / KI_B);
|
||||
}
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
fn backpressure_example() {
|
||||
println!("--- Backpressure Control ---");
|
||||
|
||||
let monitor = BackpressureMonitor::with_defaults();
|
||||
|
||||
// Check initial state
|
||||
let state = monitor.state();
|
||||
let state_str = match state {
|
||||
BackpressureState::Normal => "Normal",
|
||||
BackpressureState::Warning => "Warning",
|
||||
BackpressureState::Critical => "Critical",
|
||||
};
|
||||
println!(" Initial state: {}", state_str);
|
||||
|
||||
// Check if active
|
||||
let is_active = monitor.is_active();
|
||||
println!(" Backpressure active: {}", is_active);
|
||||
|
||||
// Try to acquire permit
|
||||
if monitor.try_acquire() {
|
||||
println!(" Successfully acquired permit");
|
||||
monitor.release();
|
||||
println!(" Released permit");
|
||||
}
|
||||
|
||||
// View statistics
|
||||
println!(" Total processed: {}", monitor.total_processed());
|
||||
println!(" Total rejected: {}", monitor.total_rejected());
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
fn deadlock_detection_example() {
|
||||
println!("--- Deadlock Detection ---");
|
||||
|
||||
let detector = DeadlockDetector::with_defaults();
|
||||
|
||||
// Register locks
|
||||
let mutex1 = detector.register_lock(LockType::Mutex);
|
||||
let mutex2 = detector.register_lock(LockType::Mutex);
|
||||
println!(" Registered locks: mutex1={}, mutex2={}", mutex1, mutex2);
|
||||
|
||||
// Simulate normal operation
|
||||
detector.record_acquire(mutex1, 1); // Thread 1 acquires mutex1
|
||||
detector.record_acquire(mutex2, 2); // Thread 2 acquires mutex2
|
||||
println!(" Normal operation: no deadlock");
|
||||
|
||||
// Detect deadlock
|
||||
if detector.detect_deadlock().is_none() {
|
||||
println!(" Detection result: no deadlock");
|
||||
}
|
||||
|
||||
// Simulate deadlock scenario
|
||||
detector.record_wait(mutex2, 1); // Thread 1 waits for mutex2
|
||||
detector.record_wait(mutex1, 2); // Thread 2 waits for mutex1
|
||||
|
||||
// Detect deadlock
|
||||
if let Some(deadlock) = detector.detect_deadlock() {
|
||||
println!(" Detection result: deadlock found {:?}", deadlock);
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
detector.unregister_lock(mutex1);
|
||||
detector.unregister_lock(mutex2);
|
||||
println!();
|
||||
}
|
||||
|
||||
fn lock_optimizer_example() {
|
||||
println!("--- Lock Optimizer ---");
|
||||
|
||||
let optimizer = LockOptimizer::with_defaults();
|
||||
|
||||
// Simulate lock operations
|
||||
for _i in 0..5 {
|
||||
optimizer.on_acquire();
|
||||
// Simulate work
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
optimizer.on_release(Duration::from_millis(10));
|
||||
}
|
||||
|
||||
// View statistics
|
||||
let stats = optimizer.stats();
|
||||
let acquired = stats.total_acquired();
|
||||
let avg_hold = stats.avg_hold_time();
|
||||
let contention = stats.contention_rate();
|
||||
|
||||
println!(" Locks acquired: {}", acquired);
|
||||
println!(" Average hold time: {:?}", avg_hold);
|
||||
println!(" Contention rate: {:.2}%", contention * 100.0);
|
||||
|
||||
println!();
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! BufReader layer optimizer for minimizing redundant buffering layers.
|
||||
//!
|
||||
//! This module provides optimization for BufReader usage in data paths,
|
||||
//! including layer count limiting and dynamic buffer size adjustment.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// BufReader optimization configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BufReaderConfig {
|
||||
/// Maximum number of nested BufReader layers (default: 2)
|
||||
pub max_layers: u32,
|
||||
|
||||
/// Buffer size for small files (default: 8KB)
|
||||
pub small_file_buffer: usize,
|
||||
|
||||
/// Buffer size for large files (default: 64KB)
|
||||
pub large_file_buffer: usize,
|
||||
|
||||
/// Threshold for large file classification (default: 1MB)
|
||||
pub large_file_threshold: usize,
|
||||
}
|
||||
|
||||
impl Default for BufReaderConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_layers: 2,
|
||||
small_file_buffer: 8 * 1024, // 8KB
|
||||
large_file_buffer: 64 * 1024, // 64KB
|
||||
large_file_threshold: 1024 * 1024, // 1MB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BufReader optimization statistics.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BufReaderStats {
|
||||
/// Total number of readers created
|
||||
pub total_readers: AtomicU64,
|
||||
|
||||
/// Number of redundant layers eliminated
|
||||
pub eliminated_layers: AtomicU64,
|
||||
|
||||
/// Number of buffer size adjustments
|
||||
pub buffer_size_adjustments: AtomicU64,
|
||||
}
|
||||
|
||||
/// BufReader layer optimizer.
|
||||
///
|
||||
/// Analyzes and optimizes BufReader nesting in data paths,
|
||||
/// dynamically adjusting buffer sizes based on data characteristics.
|
||||
pub struct BufReaderOptimizer {
|
||||
config: BufReaderConfig,
|
||||
stats: BufReaderStats,
|
||||
}
|
||||
|
||||
impl BufReaderOptimizer {
|
||||
/// Create a new BufReader optimizer with the given configuration.
|
||||
pub fn new(config: BufReaderConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
stats: BufReaderStats::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new BufReader optimizer with default configuration.
|
||||
pub fn with_defaults() -> Self {
|
||||
Self::new(BufReaderConfig::default())
|
||||
}
|
||||
|
||||
/// Calculate the optimal buffer size based on data size.
|
||||
///
|
||||
/// Returns the appropriate buffer size based on whether the data
|
||||
/// is classified as a small or large file.
|
||||
pub fn optimal_buffer_size(&self, data_size: Option<usize>) -> usize {
|
||||
match data_size {
|
||||
Some(size) if size >= self.config.large_file_threshold => self.config.large_file_buffer,
|
||||
Some(_) => self.config.small_file_buffer,
|
||||
None => self.config.small_file_buffer,
|
||||
}
|
||||
}
|
||||
|
||||
/// Optimize a reader by wrapping it with an appropriately sized BufReader.
|
||||
///
|
||||
/// This method applies the optimal buffer size based on the expected
|
||||
/// data size and tracks statistics.
|
||||
pub fn optimize<R: tokio::io::AsyncRead + Unpin>(&self, reader: R, data_size: Option<usize>) -> tokio::io::BufReader<R> {
|
||||
let buffer_size = self.optimal_buffer_size(data_size);
|
||||
self.stats.total_readers.fetch_add(1, Ordering::Relaxed);
|
||||
tokio::io::BufReader::with_capacity(buffer_size, reader)
|
||||
}
|
||||
|
||||
/// Get the statistics for this optimizer.
|
||||
pub fn stats(&self) -> &BufReaderStats {
|
||||
&self.stats
|
||||
}
|
||||
|
||||
/// Get the configuration for this optimizer.
|
||||
pub fn config(&self) -> &BufReaderConfig {
|
||||
&self.config
|
||||
}
|
||||
}
|
||||
|
||||
/// Marker trait for buffered sources.
|
||||
///
|
||||
/// Types implementing this trait are considered already buffered
|
||||
/// and should not be wrapped with additional BufReader layers.
|
||||
pub trait BufferedSource: tokio::io::AsyncRead {}
|
||||
|
||||
impl BufReaderOptimizer {
|
||||
/// Check if a reader is already a buffered source.
|
||||
///
|
||||
/// Returns true if the reader implements `BufferedSource`,
|
||||
/// indicating it should not be wrapped with BufReader.
|
||||
pub fn is_buffered_source<R: BufferedSource + ?Sized>(&self, _reader: &R) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Eliminate redundant BufReader layers if possible.
|
||||
///
|
||||
/// This method attempts to reduce the nesting depth of BufReader
|
||||
/// layers to improve performance.
|
||||
pub fn eliminate_redundant_layers<R: tokio::io::AsyncRead + Unpin>(&self, reader: R) -> R {
|
||||
// For now, just return the reader as-is
|
||||
// Future implementation could detect and unwrap nested BufReaders
|
||||
self.stats.eliminated_layers.fetch_add(0, Ordering::Relaxed);
|
||||
reader
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[test]
|
||||
fn test_default_config() {
|
||||
let config = BufReaderConfig::default();
|
||||
assert_eq!(config.max_layers, 2);
|
||||
assert_eq!(config.small_file_buffer, 8 * 1024);
|
||||
assert_eq!(config.large_file_buffer, 64 * 1024);
|
||||
assert_eq!(config.large_file_threshold, 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_optimal_buffer_size_small_file() {
|
||||
let optimizer = BufReaderOptimizer::with_defaults();
|
||||
|
||||
// Small file (< 1MB)
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(100)), 8 * 1024);
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(1024)), 8 * 1024);
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(512 * 1024)), 8 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_optimal_buffer_size_large_file() {
|
||||
let optimizer = BufReaderOptimizer::with_defaults();
|
||||
|
||||
// Large file (>= 1MB)
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(1024 * 1024)), 64 * 1024);
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(10 * 1024 * 1024)), 64 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_optimal_buffer_size_unknown() {
|
||||
let optimizer = BufReaderOptimizer::with_defaults();
|
||||
|
||||
// Unknown size
|
||||
assert_eq!(optimizer.optimal_buffer_size(None), 8 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_creates_bufreader() {
|
||||
let optimizer = BufReaderOptimizer::with_defaults();
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
let cursor = std::io::Cursor::new(data.clone());
|
||||
|
||||
let mut reader = optimizer.optimize(cursor, Some(5));
|
||||
|
||||
let mut buf = vec![0u8; 5];
|
||||
let n = reader.read(&mut buf).await.unwrap();
|
||||
|
||||
assert_eq!(n, 5);
|
||||
assert_eq!(buf, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stats_tracking() {
|
||||
let optimizer = BufReaderOptimizer::with_defaults();
|
||||
|
||||
assert_eq!(optimizer.stats().total_readers.load(Ordering::Relaxed), 0);
|
||||
|
||||
let cursor = std::io::Cursor::new(vec![1u8, 2, 3]);
|
||||
let _reader = optimizer.optimize(cursor, Some(3));
|
||||
|
||||
assert_eq!(optimizer.stats().total_readers.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_config() {
|
||||
let config = BufReaderConfig {
|
||||
max_layers: 3,
|
||||
small_file_buffer: 4 * 1024,
|
||||
large_file_buffer: 128 * 1024,
|
||||
large_file_threshold: 2 * 1024 * 1024,
|
||||
};
|
||||
|
||||
let optimizer = BufReaderOptimizer::new(config);
|
||||
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(1024 * 1024)), 4 * 1024);
|
||||
assert_eq!(optimizer.optimal_buffer_size(Some(3 * 1024 * 1024)), 128 * 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Aligned pread-based file reader.
|
||||
//!
|
||||
//! This module provides an aligned, position-based file reader that uses
|
||||
//! `pread`/`FileExt::read_at` for I/O operations. It performs reads at
|
||||
//! 512-byte-aligned offsets and sizes, making it suitable as a foundation
|
||||
//! for workloads where alignment matters.
|
||||
//!
|
||||
//! Note: This reader does **not** set the `O_DIRECT` flag and therefore does
|
||||
//! not bypass the OS page cache. It is an aligned `pread`-based reader, not
|
||||
//! true Direct I/O. To implement true O_DIRECT on Linux, the file must be
|
||||
//! opened with `O_DIRECT` via `libc::open`.
|
||||
//!
|
||||
//! # Platform Support
|
||||
//!
|
||||
//! The `read_at` implementation is only available on Unix-like platforms.
|
||||
//! On other platforms, this reader will return an error.
|
||||
|
||||
use std::io::{self};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// Errors that can occur during aligned pread operations.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum AlignedPreadError {
|
||||
/// Platform doesn't support `read_at`-based I/O
|
||||
UnsupportedPlatform,
|
||||
/// File descriptor doesn't support this reader
|
||||
UnsupportedFile,
|
||||
/// I/O error occurred
|
||||
Io(String),
|
||||
/// Invalid alignment (reads require 512-byte-aligned offset and size)
|
||||
AlignmentError { offset: u64, size: usize },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AlignedPreadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::UnsupportedPlatform => write!(f, "Aligned pread not supported on this platform"),
|
||||
Self::UnsupportedFile => write!(f, "File doesn't support this reader"),
|
||||
Self::Io(msg) => write!(f, "I/O error: {}", msg),
|
||||
Self::AlignmentError { offset, size } => {
|
||||
write!(f, "Alignment error: offset={}, size={}", offset, size)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AlignedPreadError {}
|
||||
|
||||
impl From<io::Error> for AlignedPreadError {
|
||||
fn from(err: io::Error) -> Self {
|
||||
Self::Io(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Aligned pread-based file reader for Unix platforms.
|
||||
///
|
||||
/// This reader performs I/O using `pread`/`FileExt::read_at` at
|
||||
/// 512-byte-aligned offsets and sizes, without modifying the file's
|
||||
/// current position.
|
||||
///
|
||||
/// **Note:** This reader does **not** set the `O_DIRECT` flag and therefore
|
||||
/// does **not** bypass the OS page cache. It is an aligned `pread`-based
|
||||
/// reader. To implement true O_DIRECT, the file must be opened with
|
||||
/// `O_DIRECT` via `libc::open`.
|
||||
///
|
||||
/// # Platform Support
|
||||
///
|
||||
/// Only available on Linux (uses `FileExt::read_at`). On other platforms,
|
||||
/// use `BytesBufferedReader` instead.
|
||||
///
|
||||
/// # Alignment Requirements
|
||||
///
|
||||
/// Reads have strict alignment requirements:
|
||||
/// - File offset must be aligned to 512 bytes
|
||||
/// - Buffer size must be a multiple of 512 bytes
|
||||
/// - Buffer address must be aligned (handled internally)
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use rustfs_io_core::AlignedPreadReader;
|
||||
///
|
||||
/// // Linux only
|
||||
/// #[cfg(target_os = "linux")]
|
||||
/// let reader = AlignedPreadReader::new(file, offset, size)?;
|
||||
/// ```
|
||||
#[cfg(target_os = "linux")]
|
||||
pub struct AlignedPreadReader {
|
||||
/// Underlying file handle used for aligned pread I/O
|
||||
file: std::fs::File,
|
||||
/// Current read position
|
||||
pos: u64,
|
||||
/// Remaining bytes to read
|
||||
remaining: usize,
|
||||
/// Buffer for aligned reads
|
||||
buffer: Vec<u8>,
|
||||
/// Current position in the buffer
|
||||
buffer_pos: usize,
|
||||
/// Amount of data in the buffer
|
||||
buffer_len: usize,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
impl AlignedPreadReader {
|
||||
/// Alignment requirement for reads (512 bytes for most systems)
|
||||
pub const ALIGNMENT: usize = 512;
|
||||
|
||||
/// Create a new aligned pread-based reader.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file` - File to read from
|
||||
/// * `offset` - Starting offset in the file (must be 512-byte aligned)
|
||||
/// * `size` - Number of bytes to read (must be 512-byte aligned)
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// An `AlignedPreadReader` that reads the file at the given offset.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if offset or size are not 512-byte aligned.
|
||||
pub fn new(file: std::fs::File, offset: u64, size: usize) -> Result<Self, AlignedPreadError> {
|
||||
// Check alignment
|
||||
if !offset.is_multiple_of(Self::ALIGNMENT as u64) {
|
||||
return Err(AlignedPreadError::AlignmentError { offset, size });
|
||||
}
|
||||
if !size.is_multiple_of(Self::ALIGNMENT) {
|
||||
return Err(AlignedPreadError::AlignmentError { offset, size });
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
pos: offset,
|
||||
remaining: size,
|
||||
buffer: Vec::new(),
|
||||
buffer_pos: 0,
|
||||
buffer_len: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a chunk of data using aligned pread.
|
||||
///
|
||||
/// This method performs aligned reads and handles the buffering required
|
||||
/// by this aligned pread implementation. It does not use `O_DIRECT`.
|
||||
fn read_chunk(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
// If buffer is exhausted, read more data
|
||||
if self.buffer_pos >= self.buffer_len {
|
||||
if self.remaining == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Allocate aligned buffer
|
||||
let chunk_size = (self.remaining).min(64 * 1024); // 64KB chunks
|
||||
let aligned_size = chunk_size.div_ceil(Self::ALIGNMENT) * Self::ALIGNMENT;
|
||||
|
||||
self.buffer = vec![0u8; aligned_size];
|
||||
|
||||
// Use pread for atomic read at position (no file offset modification)
|
||||
use std::os::unix::fs::FileExt;
|
||||
let n = self.file.read_at(&mut self.buffer, self.pos)?;
|
||||
|
||||
self.buffer_pos = 0;
|
||||
self.buffer_len = n;
|
||||
self.pos += n as u64;
|
||||
self.remaining -= n;
|
||||
|
||||
if n == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Copy from buffer to user buffer
|
||||
let available = self.buffer_len - self.buffer_pos;
|
||||
let to_copy = buf.len().min(available);
|
||||
buf[..to_copy].copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + to_copy]);
|
||||
self.buffer_pos += to_copy;
|
||||
|
||||
Ok(to_copy)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
impl AsyncRead for AlignedPreadReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let filled = buf.filled().len();
|
||||
let mut remaining = buf.initialize_unfilled();
|
||||
|
||||
while !remaining.is_empty() {
|
||||
match self.read_chunk(remaining) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
remaining = &mut remaining[n..];
|
||||
}
|
||||
Err(e) => return Poll::Ready(Err(e)),
|
||||
}
|
||||
}
|
||||
|
||||
let _n_read = buf.filled().len() - filled;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Aligned pread reader stub for non-Linux platforms.
|
||||
///
|
||||
/// On non-Linux platforms, `read_at`-based I/O is not available through this
|
||||
/// type. This stub exists to provide a consistent API across platforms.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub struct AlignedPreadReader {
|
||||
_priv: (),
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
impl AlignedPreadReader {
|
||||
/// Create a new aligned pread reader (not supported on this platform).
|
||||
///
|
||||
/// Always returns an error on non-Linux platforms.
|
||||
pub fn new(_file: std::fs::File, _offset: u64, _size: usize) -> Result<Self, AlignedPreadError> {
|
||||
Err(AlignedPreadError::UnsupportedPlatform)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
impl AsyncRead for AlignedPreadReader {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"Aligned pread-based I/O not supported on this platform",
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AlignedPreadReader {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
f.debug_struct("AlignedPreadReader")
|
||||
.field("pos", &self.pos)
|
||||
.field("remaining", &self.remaining)
|
||||
.field("buffer_len", &self.buffer_len)
|
||||
.finish()
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
f.debug_struct("AlignedPreadReader")
|
||||
.field("platform", &"unsupported")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Historical name for aligned pread errors.
|
||||
#[deprecated(since = "1.0.0-beta.8", note = "use AlignedPreadError; this reader does not set O_DIRECT")]
|
||||
pub type DirectIoError = AlignedPreadError;
|
||||
|
||||
/// Historical name for the aligned pread-based reader.
|
||||
#[deprecated(since = "1.0.0-beta.8", note = "use AlignedPreadReader; this reader does not set O_DIRECT")]
|
||||
pub type DirectIoReader = AlignedPreadReader;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_alignment_check() {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// Valid alignment
|
||||
let file = std::fs::File::open("/dev/zero").unwrap();
|
||||
assert!(
|
||||
AlignedPreadReader::new(file, 0, 512).is_ok(),
|
||||
"Should succeed with aligned offset and size"
|
||||
);
|
||||
|
||||
let file = std::fs::File::open("/dev/zero").expect("open /dev/zero for alias");
|
||||
assert!(
|
||||
AlignedPreadReader::new(file, 0, 512).is_ok(),
|
||||
"Should succeed through aligned pread alias"
|
||||
);
|
||||
|
||||
// Invalid offset
|
||||
let file = std::fs::File::open("/dev/zero").unwrap();
|
||||
assert!(AlignedPreadReader::new(file, 1, 512).is_err(), "Should fail with unaligned offset");
|
||||
|
||||
// Invalid size
|
||||
let file = std::fs::File::open("/dev/zero").unwrap();
|
||||
assert!(AlignedPreadReader::new(file, 0, 511).is_err(), "Should fail with unaligned size");
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
// Non-Linux should return UnsupportedPlatform
|
||||
let file = std::fs::File::open(std::env::current_exe().unwrap()).unwrap();
|
||||
assert!(matches!(
|
||||
AlignedPreadReader::new(file, 0, 512),
|
||||
Err(AlignedPreadError::UnsupportedPlatform)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn test_legacy_direct_io_alias() {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let file = std::fs::File::open("/dev/zero").unwrap();
|
||||
assert!(DirectIoReader::new(file, 0, 512).is_ok());
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let file = std::fs::File::open(std::env::current_exe().unwrap()).unwrap();
|
||||
assert!(matches!(DirectIoReader::new(file, 0, 512), Err(AlignedPreadError::UnsupportedPlatform)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! I/O priority queue for scheduling I/O operations.
|
||||
//!
|
||||
//! This module provides a priority queue implementation for I/O operations
|
||||
//! with support for starvation prevention and fair scheduling.
|
||||
|
||||
use crate::config::IoPriorityQueueConfig;
|
||||
use crate::scheduler::IoPriority;
|
||||
use std::collections::VecDeque;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// A queued I/O request.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IoRequest {
|
||||
/// Request ID.
|
||||
pub id: u64,
|
||||
/// Request priority.
|
||||
pub priority: IoPriority,
|
||||
/// Request size in bytes.
|
||||
pub size: usize,
|
||||
/// Queue time.
|
||||
pub queued_at: Instant,
|
||||
/// Whether this is a sequential read.
|
||||
pub is_sequential: bool,
|
||||
}
|
||||
|
||||
impl IoRequest {
|
||||
/// Create a new I/O request.
|
||||
pub fn new(id: u64, priority: IoPriority, size: usize, is_sequential: bool) -> Self {
|
||||
Self {
|
||||
id,
|
||||
priority,
|
||||
size,
|
||||
queued_at: Instant::now(),
|
||||
is_sequential,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the wait time in the queue.
|
||||
pub fn wait_time(&self) -> Duration {
|
||||
self.queued_at.elapsed()
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue status for a priority level.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct IoQueueStatus {
|
||||
/// Number of requests in the queue.
|
||||
pub count: usize,
|
||||
/// Total size of all requests.
|
||||
pub total_size: usize,
|
||||
/// Oldest request wait time.
|
||||
pub oldest_wait: Option<Duration>,
|
||||
/// Number of requests processed.
|
||||
pub processed: u64,
|
||||
}
|
||||
|
||||
impl IoQueueStatus {
|
||||
/// Create new queue status.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// I/O priority queue.
|
||||
pub struct IoPriorityQueue {
|
||||
/// Queue configuration.
|
||||
config: IoPriorityQueueConfig,
|
||||
/// High priority queue.
|
||||
high: VecDeque<IoRequest>,
|
||||
/// Normal priority queue.
|
||||
normal: VecDeque<IoRequest>,
|
||||
/// Low priority queue.
|
||||
low: VecDeque<IoRequest>,
|
||||
/// Next request ID.
|
||||
next_id: u64,
|
||||
/// Last dequeue time for each priority (for starvation prevention).
|
||||
last_dequeue: [Option<Instant>; 3],
|
||||
/// Statistics for each queue.
|
||||
stats: [IoQueueStatus; 3],
|
||||
}
|
||||
|
||||
impl IoPriorityQueue {
|
||||
/// Create a new priority queue with the given configuration.
|
||||
pub fn new(config: IoPriorityQueueConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
high: VecDeque::with_capacity(100),
|
||||
normal: VecDeque::with_capacity(500),
|
||||
low: VecDeque::with_capacity(200),
|
||||
next_id: 0,
|
||||
last_dequeue: [None, None, None],
|
||||
stats: [IoQueueStatus::new(), IoQueueStatus::new(), IoQueueStatus::new()],
|
||||
}
|
||||
}
|
||||
|
||||
/// Create with default configuration.
|
||||
pub fn with_defaults() -> Self {
|
||||
Self::new(IoPriorityQueueConfig::default())
|
||||
}
|
||||
|
||||
/// Get the configuration.
|
||||
pub fn config(&self) -> &IoPriorityQueueConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Enqueue a request.
|
||||
pub fn enqueue(&mut self, priority: IoPriority, size: usize, is_sequential: bool) -> u64 {
|
||||
let id = self.next_id;
|
||||
self.next_id += 1;
|
||||
|
||||
let request = IoRequest::new(id, priority, size, is_sequential);
|
||||
|
||||
match priority {
|
||||
IoPriority::High => {
|
||||
if self.high.len() < self.config.high_capacity {
|
||||
self.high.push_back(request);
|
||||
}
|
||||
}
|
||||
IoPriority::Normal => {
|
||||
if self.normal.len() < self.config.normal_capacity {
|
||||
self.normal.push_back(request);
|
||||
}
|
||||
}
|
||||
IoPriority::Low => {
|
||||
if self.low.len() < self.config.low_capacity {
|
||||
self.low.push_back(request);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
id
|
||||
}
|
||||
|
||||
/// Dequeue the next request.
|
||||
///
|
||||
/// Uses weighted fair queuing with starvation prevention.
|
||||
pub fn dequeue(&mut self) -> Option<IoRequest> {
|
||||
let now = Instant::now();
|
||||
|
||||
// Check for starvation: if a lower priority queue hasn't been served in a while,
|
||||
// give it priority
|
||||
let normal_starved = self.is_starved(IoPriority::Normal, now);
|
||||
let low_starved = self.is_starved(IoPriority::Low, now);
|
||||
|
||||
// Priority order with starvation consideration
|
||||
// Check conditions first, then dequeue
|
||||
let dequeue_high = !self.high.is_empty() && !low_starved && !normal_starved;
|
||||
let dequeue_normal = !self.normal.is_empty() && !low_starved;
|
||||
let dequeue_low = !self.low.is_empty();
|
||||
let dequeue_high_fallback = !self.high.is_empty();
|
||||
let dequeue_normal_fallback = !self.normal.is_empty();
|
||||
|
||||
if dequeue_high {
|
||||
let request = self.high.pop_front();
|
||||
if request.is_some() {
|
||||
self.last_dequeue[0] = Some(Instant::now());
|
||||
self.stats[0].processed += 1;
|
||||
}
|
||||
request
|
||||
} else if dequeue_normal {
|
||||
let request = self.normal.pop_front();
|
||||
if request.is_some() {
|
||||
self.last_dequeue[1] = Some(Instant::now());
|
||||
self.stats[1].processed += 1;
|
||||
}
|
||||
request
|
||||
} else if dequeue_low {
|
||||
let request = self.low.pop_front();
|
||||
if request.is_some() {
|
||||
self.last_dequeue[2] = Some(Instant::now());
|
||||
self.stats[2].processed += 1;
|
||||
}
|
||||
request
|
||||
} else if dequeue_high_fallback {
|
||||
let request = self.high.pop_front();
|
||||
if request.is_some() {
|
||||
self.last_dequeue[0] = Some(Instant::now());
|
||||
self.stats[0].processed += 1;
|
||||
}
|
||||
request
|
||||
} else if dequeue_normal_fallback {
|
||||
let request = self.normal.pop_front();
|
||||
if request.is_some() {
|
||||
self.last_dequeue[1] = Some(Instant::now());
|
||||
self.stats[1].processed += 1;
|
||||
}
|
||||
request
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a priority level is starved.
|
||||
fn is_starved(&self, priority: IoPriority, now: Instant) -> bool {
|
||||
let idx = match priority {
|
||||
IoPriority::High => 0,
|
||||
IoPriority::Normal => 1,
|
||||
IoPriority::Low => 2,
|
||||
};
|
||||
|
||||
if let Some(last) = self.last_dequeue[idx] {
|
||||
now.duration_since(last) > self.config.starvation_threshold
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the total number of queued requests.
|
||||
pub fn len(&self) -> usize {
|
||||
self.high.len() + self.normal.len() + self.low.len()
|
||||
}
|
||||
|
||||
/// Check if the queue is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.high.is_empty() && self.normal.is_empty() && self.low.is_empty()
|
||||
}
|
||||
|
||||
/// Get queue status for a priority level.
|
||||
pub fn status(&self, priority: IoPriority) -> IoQueueStatus {
|
||||
let (queue, idx) = match priority {
|
||||
IoPriority::High => (&self.high, 0),
|
||||
IoPriority::Normal => (&self.normal, 1),
|
||||
IoPriority::Low => (&self.low, 2),
|
||||
};
|
||||
|
||||
let mut status = self.stats[idx].clone();
|
||||
status.count = queue.len();
|
||||
status.total_size = queue.iter().map(|r| r.size).sum();
|
||||
status.oldest_wait = queue.front().map(|r| r.wait_time());
|
||||
status
|
||||
}
|
||||
|
||||
/// Get the total queue status.
|
||||
pub fn total_status(&self) -> IoQueueStatus {
|
||||
let mut total = IoQueueStatus::new();
|
||||
total.count = self.len();
|
||||
total.total_size = self
|
||||
.high
|
||||
.iter()
|
||||
.chain(self.normal.iter())
|
||||
.chain(self.low.iter())
|
||||
.map(|r| r.size)
|
||||
.sum();
|
||||
total.processed = self.stats.iter().map(|s| s.processed).sum();
|
||||
total.oldest_wait = self
|
||||
.high
|
||||
.front()
|
||||
.map(|r| r.wait_time())
|
||||
.or_else(|| self.normal.front().map(|r| r.wait_time()))
|
||||
.or_else(|| self.low.front().map(|r| r.wait_time()));
|
||||
total
|
||||
}
|
||||
|
||||
/// Clear all queues.
|
||||
pub fn clear(&mut self) {
|
||||
self.high.clear();
|
||||
self.normal.clear();
|
||||
self.low.clear();
|
||||
}
|
||||
|
||||
/// Peek at the next request without removing it.
|
||||
pub fn peek(&self) -> Option<&IoRequest> {
|
||||
if !self.high.is_empty() {
|
||||
self.high.front()
|
||||
} else if !self.normal.is_empty() {
|
||||
self.normal.front()
|
||||
} else {
|
||||
self.low.front()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_enqueue_dequeue() {
|
||||
let mut queue = IoPriorityQueue::with_defaults();
|
||||
|
||||
let id1 = queue.enqueue(IoPriority::High, 1024, true);
|
||||
let id2 = queue.enqueue(IoPriority::Normal, 2048, false);
|
||||
let id3 = queue.enqueue(IoPriority::Low, 4096, true);
|
||||
|
||||
assert_eq!(queue.len(), 3);
|
||||
|
||||
// High priority should be dequeued first
|
||||
let req1 = queue.dequeue().unwrap();
|
||||
assert_eq!(req1.id, id1);
|
||||
assert_eq!(req1.priority, IoPriority::High);
|
||||
|
||||
let req2 = queue.dequeue().unwrap();
|
||||
assert_eq!(req2.id, id2);
|
||||
assert_eq!(req2.priority, IoPriority::Normal);
|
||||
|
||||
let req3 = queue.dequeue().unwrap();
|
||||
assert_eq!(req3.id, id3);
|
||||
assert_eq!(req3.priority, IoPriority::Low);
|
||||
|
||||
assert!(queue.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_queue_status() {
|
||||
let mut queue = IoPriorityQueue::with_defaults();
|
||||
|
||||
queue.enqueue(IoPriority::High, 1024, true);
|
||||
queue.enqueue(IoPriority::High, 2048, true);
|
||||
queue.enqueue(IoPriority::Normal, 4096, false);
|
||||
|
||||
let high_status = queue.status(IoPriority::High);
|
||||
assert_eq!(high_status.count, 2);
|
||||
assert_eq!(high_status.total_size, 3072);
|
||||
|
||||
let normal_status = queue.status(IoPriority::Normal);
|
||||
assert_eq!(normal_status.count, 1);
|
||||
assert_eq!(normal_status.total_size, 4096);
|
||||
|
||||
let total = queue.total_status();
|
||||
assert_eq!(total.count, 3);
|
||||
assert_eq!(total.total_size, 7168);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_queue_capacity() {
|
||||
let config = IoPriorityQueueConfig {
|
||||
high_capacity: 2,
|
||||
normal_capacity: 2,
|
||||
low_capacity: 2,
|
||||
..Default::default()
|
||||
};
|
||||
let mut queue = IoPriorityQueue::new(config);
|
||||
|
||||
queue.enqueue(IoPriority::High, 1024, true);
|
||||
queue.enqueue(IoPriority::High, 1024, true);
|
||||
queue.enqueue(IoPriority::High, 1024, true); // Should be dropped
|
||||
|
||||
assert_eq!(queue.status(IoPriority::High).count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clear() {
|
||||
let mut queue = IoPriorityQueue::with_defaults();
|
||||
|
||||
queue.enqueue(IoPriority::High, 1024, true);
|
||||
queue.enqueue(IoPriority::Normal, 2048, false);
|
||||
queue.enqueue(IoPriority::Low, 4096, true);
|
||||
|
||||
assert_eq!(queue.len(), 3);
|
||||
queue.clear();
|
||||
assert!(queue.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peek() {
|
||||
let mut queue = IoPriorityQueue::with_defaults();
|
||||
|
||||
queue.enqueue(IoPriority::Normal, 2048, false);
|
||||
queue.enqueue(IoPriority::High, 1024, true);
|
||||
|
||||
let peeked = queue.peek().unwrap();
|
||||
assert_eq!(peeked.priority, IoPriority::High);
|
||||
|
||||
// Peek shouldn't remove the item
|
||||
assert_eq!(queue.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ pub enum StorageMedia {
|
||||
}
|
||||
|
||||
impl StorageMedia {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Nvme => "nvme",
|
||||
@@ -59,6 +60,7 @@ pub enum AccessPattern {
|
||||
}
|
||||
|
||||
impl AccessPattern {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Sequential => "sequential",
|
||||
@@ -69,21 +71,25 @@ impl AccessPattern {
|
||||
}
|
||||
|
||||
/// Check if this is a sequential access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_sequential(&self) -> bool {
|
||||
matches!(self, Self::Sequential)
|
||||
}
|
||||
|
||||
/// Check if this is a random access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_random(&self) -> bool {
|
||||
matches!(self, Self::Random)
|
||||
}
|
||||
|
||||
/// Check if this is a mixed access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_mixed(&self) -> bool {
|
||||
matches!(self, Self::Mixed)
|
||||
}
|
||||
|
||||
/// Check if this pattern is unknown.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_unknown(&self) -> bool {
|
||||
matches!(self, Self::Unknown)
|
||||
}
|
||||
|
||||
+61
-12
@@ -12,39 +12,85 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared I/O primitives for RustFS.
|
||||
//! Buffered I/O reader and writer implementations for RustFS.
|
||||
//!
|
||||
//! This crate holds the buffer pool and the concurrency-control primitives
|
||||
//! that the storage layer builds on:
|
||||
//! This crate provides buffered readers and writers for I/O operations.
|
||||
//! Prefer `BytesBufferedReader`, `BytesMutWriter`, and `AlignedPreadReader`
|
||||
//! for new code. Historical `ZeroCopy*` and `DirectIo*` names remain exported
|
||||
//! for backward compatibility.
|
||||
//!
|
||||
//! - Tiered `BytesPool` for buffer management
|
||||
//! - Storage-media and access-pattern profiling (`io_profile`)
|
||||
//! - Scheduler and priority-queue configuration shapes
|
||||
//! - Backpressure admission, deadlock detection, lock optimization
|
||||
//! - Progress tracking for long-running operations
|
||||
//! # Features
|
||||
//!
|
||||
//! - Memory-mapped file reading (mmap-then-copy) on Unix platforms
|
||||
//! - Bytes-based buffered wrapping
|
||||
//! - AsyncRead trait implementations
|
||||
//! - Tiered BytesPool for buffer management
|
||||
//! - Aligned pread-based reader (NOT true Direct I/O / O_DIRECT)
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use rustfs_io_core::BytesPool;
|
||||
//! use rustfs_io_core::{BytesBufferedReader, BytesPool};
|
||||
//! use bytes::Bytes;
|
||||
//!
|
||||
//! // Create from existing bytes (zero-copy)
|
||||
//! let data = Bytes::from("hello world");
|
||||
//! let reader = BytesBufferedReader::from_bytes(data);
|
||||
//!
|
||||
//! // Create from file using buffered reads
|
||||
//! let reader = BytesBufferedReader::from_file_read(&file, 0, 1024).await?;
|
||||
//!
|
||||
//! // Use BytesPool
|
||||
//! let pool = BytesPool::new_tiered();
|
||||
//! let mut buffer = pool.acquire_buffer(8192).await;
|
||||
//! ```
|
||||
|
||||
pub mod backpressure;
|
||||
pub mod bufreader_optimizer;
|
||||
pub mod config;
|
||||
pub mod deadlock_detector;
|
||||
pub mod direct_io;
|
||||
pub mod io_priority_queue;
|
||||
pub mod io_profile;
|
||||
pub mod lock_optimizer;
|
||||
pub mod pool;
|
||||
pub mod progress;
|
||||
pub mod reader;
|
||||
pub mod scheduler;
|
||||
pub mod shared_memory;
|
||||
pub mod timeout_wrapper;
|
||||
pub mod writer;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub use direct_io::{AlignedPreadError, AlignedPreadReader};
|
||||
#[cfg(target_os = "linux")]
|
||||
#[allow(deprecated)]
|
||||
pub use direct_io::{DirectIoError, DirectIoReader};
|
||||
pub use pool::{BytesPool, BytesPoolConfig, BytesPoolMetrics, PooledBuffer};
|
||||
#[allow(deprecated)]
|
||||
pub use reader::ZeroCopyObjectReader;
|
||||
pub use reader::{BytesBufferedReader, ZeroCopyReadError};
|
||||
#[allow(deprecated)]
|
||||
pub use writer::ZeroCopyObjectWriter;
|
||||
pub use writer::{BytesMutWriter, ZeroCopyWriteError};
|
||||
|
||||
// BufReader optimizer exports
|
||||
pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource};
|
||||
|
||||
// Shared memory exports
|
||||
pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats};
|
||||
|
||||
// Config exports
|
||||
pub use config::{ConfigError, IoPriorityQueueConfig, IoSchedulerConfig};
|
||||
|
||||
// Scheduler exports
|
||||
pub use scheduler::{
|
||||
BandwidthTier, IoLoadLevel, IoLoadMetrics, IoPriority, IoScheduler, IoSchedulingContext, IoStrategy, KI_B, MI_B,
|
||||
calculate_optimal_buffer_size, get_advanced_buffer_size, get_buffer_size_for_media, get_concurrency_aware_buffer_size,
|
||||
};
|
||||
|
||||
// Priority queue exports
|
||||
pub use io_priority_queue::{IoPriorityQueue, IoQueueStatus, IoRequest};
|
||||
|
||||
// Backpressure exports
|
||||
pub use backpressure::{BackpressureConfig, BackpressureError, BackpressureMonitor, BackpressureState};
|
||||
|
||||
@@ -54,5 +100,8 @@ pub use deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig, LockInfo,
|
||||
// Lock optimizer exports
|
||||
pub use lock_optimizer::{LockGuard, LockOptimizeConfig, LockOptimizer, LockStats};
|
||||
|
||||
// Progress tracking exports
|
||||
pub use progress::OperationProgress;
|
||||
// Timeout wrapper exports
|
||||
pub use timeout_wrapper::{
|
||||
OperationProgress, RequestTimeoutWrapper, TimeoutConfig, TimeoutError, TimeoutStats, calculate_adaptive_timeout,
|
||||
estimate_bytes_per_second,
|
||||
};
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Progress tracking for long-running I/O operations.
|
||||
//!
|
||||
//! Re-exported as `rustfs_concurrency::OperationProgress` for the storage
|
||||
//! timeout implementation, which uses `is_stale` to tell a slow transfer
|
||||
//! apart from a stalled one.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Operation progress tracker.
|
||||
#[derive(Debug)]
|
||||
pub struct OperationProgress {
|
||||
/// Total size (if known).
|
||||
pub total_size: Option<u64>,
|
||||
/// Bytes processed.
|
||||
bytes_processed: AtomicU64,
|
||||
/// Last update time.
|
||||
last_update: std::sync::Mutex<Instant>,
|
||||
/// Stale timeout.
|
||||
stale_timeout: Duration,
|
||||
/// Start time for transfer rate calculation.
|
||||
start_time: Instant,
|
||||
}
|
||||
|
||||
impl OperationProgress {
|
||||
/// Create new operation progress.
|
||||
pub fn new(total_size: Option<u64>, stale_timeout: Duration) -> Self {
|
||||
Self {
|
||||
total_size,
|
||||
bytes_processed: AtomicU64::new(0),
|
||||
last_update: std::sync::Mutex::new(Instant::now()),
|
||||
stale_timeout,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Update progress.
|
||||
pub fn update(&self, bytes: u64) {
|
||||
self.bytes_processed.store(bytes, Ordering::Relaxed);
|
||||
if let Ok(mut last) = self.last_update.lock() {
|
||||
*last = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Add to progress.
|
||||
pub fn add(&self, bytes: u64) {
|
||||
self.bytes_processed.fetch_add(bytes, Ordering::Relaxed);
|
||||
if let Ok(mut last) = self.last_update.lock() {
|
||||
*last = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current progress.
|
||||
pub fn current(&self) -> u64 {
|
||||
self.bytes_processed.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Check if progress is stale.
|
||||
pub fn is_stale(&self) -> bool {
|
||||
if let Ok(last) = self.last_update.lock() {
|
||||
last.elapsed() > self.stale_timeout
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Get progress percentage.
|
||||
pub fn progress_percent(&self) -> Option<f64> {
|
||||
self.total_size.map(|total| {
|
||||
if total == 0 {
|
||||
100.0
|
||||
} else {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
(processed as f64 / total as f64 * 100.0).min(100.0)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Get remaining bytes.
|
||||
pub fn remaining(&self) -> Option<u64> {
|
||||
self.total_size.map(|total| {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
total.saturating_sub(processed)
|
||||
})
|
||||
}
|
||||
|
||||
/// Calculate transfer rate in bytes per second.
|
||||
///
|
||||
/// Returns 0 if no time has elapsed or no data transferred.
|
||||
pub fn transfer_rate(&self) -> u64 {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
if processed == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let elapsed = self.start_time.elapsed().as_secs_f64();
|
||||
if elapsed > 0.0 {
|
||||
(processed as f64 / elapsed) as u64
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_operation_progress() {
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
|
||||
assert_eq!(progress.current(), 0);
|
||||
assert_eq!(progress.progress_percent(), Some(0.0));
|
||||
|
||||
progress.update(500);
|
||||
assert_eq!(progress.current(), 500);
|
||||
assert_eq!(progress.progress_percent(), Some(50.0));
|
||||
|
||||
progress.add(300);
|
||||
assert_eq!(progress.current(), 800);
|
||||
assert_eq!(progress.remaining(), Some(200));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bytes-backed object reader implementation.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// Errors that can occur during Bytes-backed read operations.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ZeroCopyReadError {
|
||||
/// I/O error occurred.
|
||||
Io(String),
|
||||
/// Memory mapping error.
|
||||
Mmap(String),
|
||||
/// Invalid offset or size.
|
||||
InvalidRange,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ZeroCopyReadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Io(msg) => write!(f, "I/O error: {}", msg),
|
||||
Self::Mmap(msg) => write!(f, "Mmap error: {}", msg),
|
||||
Self::InvalidRange => write!(f, "Invalid offset or size"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ZeroCopyReadError {}
|
||||
|
||||
impl From<io::Error> for ZeroCopyReadError {
|
||||
fn from(err: io::Error) -> Self {
|
||||
Self::Io(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes-backed object reader.
|
||||
///
|
||||
/// `from_bytes` wraps existing `Bytes` without copying, but file constructors
|
||||
/// copy file data into owned `Bytes` after mmap or normal reads.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use bytes::Bytes;
|
||||
/// use rustfs_io_core::BytesBufferedReader;
|
||||
///
|
||||
/// // Create from bytes without copying the `Bytes` buffer
|
||||
/// let data = Bytes::from("hello world");
|
||||
/// let reader = BytesBufferedReader::from_bytes(data);
|
||||
///
|
||||
/// // Read using AsyncRead trait
|
||||
/// let mut buf = vec![0u8; 1024];
|
||||
/// let n = reader.read(&mut buf[..]).await?;
|
||||
/// ```
|
||||
pub struct BytesBufferedReader {
|
||||
/// Internal data source (could be mmap or owned bytes)
|
||||
data: Bytes,
|
||||
/// Current read position
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
/// Historical name for the bytes-backed object reader.
|
||||
#[deprecated(
|
||||
since = "1.0.0-beta.8",
|
||||
note = "use BytesBufferedReader; file constructors copy into owned Bytes"
|
||||
)]
|
||||
pub type ZeroCopyObjectReader = BytesBufferedReader;
|
||||
|
||||
impl BytesBufferedReader {
|
||||
/// Create a reader from existing bytes.
|
||||
///
|
||||
/// This is a true zero-copy operation - the Bytes are wrapped
|
||||
/// without any allocation or copying.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data` - Bytes to wrap
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let data = Bytes::from("hello world");
|
||||
/// let reader = BytesBufferedReader::from_bytes(data);
|
||||
/// ```
|
||||
pub fn from_bytes(data: Bytes) -> Self {
|
||||
Self { data, pos: 0 }
|
||||
}
|
||||
|
||||
/// Create a Bytes-backed reader from a file using mmap-then-copy.
|
||||
///
|
||||
/// This maps the requested file range and copies it into owned `Bytes`
|
||||
/// before returning. It does not expose the mmap as a zero-copy buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - Path to the file to memory map
|
||||
/// * `offset` - Offset within the file to start reading
|
||||
/// * `size` - Number of bytes to read
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the file cannot be memory mapped.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let reader = BytesBufferedReader::from_file_mmap_path("large_file.bin", 0, 1024).await?;
|
||||
/// ```
|
||||
#[cfg(unix)]
|
||||
// SAFETY: The mmap is created from a read-only file handle for the
|
||||
// caller-provided range, then copied into owned `Bytes` before the file and
|
||||
// mapping are dropped.
|
||||
#[allow(unsafe_code)]
|
||||
pub async fn from_file_mmap_path(path: &std::path::Path, offset: u64, size: usize) -> Result<Self, ZeroCopyReadError> {
|
||||
use memmap2::MmapOptions;
|
||||
|
||||
let path = path.to_path_buf();
|
||||
let (offset, size) = (offset, size);
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Open the file in sync context
|
||||
let std_file = std::fs::File::open(&path).map_err(|e| ZeroCopyReadError::Io(e.to_string()))?;
|
||||
|
||||
// SAFETY: `std_file` remains open while the mapping is created and
|
||||
// copied, and the mapped bytes are not exposed beyond this closure.
|
||||
let mmap = unsafe { MmapOptions::new().offset(offset).len(size).map(&std_file) }
|
||||
.map_err(|e| ZeroCopyReadError::Mmap(e.to_string()))?;
|
||||
|
||||
// Convert to Bytes (this is a copy, but only done once)
|
||||
Ok(Self {
|
||||
data: Bytes::copy_from_slice(&mmap),
|
||||
pos: 0,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ZeroCopyReadError::Io(e.to_string()))?
|
||||
}
|
||||
|
||||
/// Create a Bytes-backed reader from a file using normal reads.
|
||||
///
|
||||
/// This path reads the requested range into an owned buffer and wraps it in
|
||||
/// `Bytes`. It does not perform mmap or zero-copy file I/O.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file` - File to read from
|
||||
/// * `offset` - Offset within the file to start reading
|
||||
/// * `size` - Number of bytes to map
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A reader backed by copied file data.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the file cannot be read.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let file = tokio::fs::File::open("large_file.bin").await?;
|
||||
/// let reader = BytesBufferedReader::from_file_read(&file, 0, 1024).await?;
|
||||
/// ```
|
||||
#[cfg(unix)]
|
||||
pub async fn from_file_read(file: &tokio::fs::File, offset: u64, size: usize) -> Result<Self, ZeroCopyReadError> {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom};
|
||||
|
||||
let mut cloned = file.try_clone().await?;
|
||||
cloned.seek(SeekFrom::Start(offset)).await?;
|
||||
|
||||
let mut buffer = vec![0u8; size];
|
||||
cloned.read_exact(&mut buffer).await?;
|
||||
|
||||
Ok(Self {
|
||||
data: Bytes::from(buffer),
|
||||
pos: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a Bytes-backed reader from a file (non-Unix fallback).
|
||||
///
|
||||
/// On platforms that don't support mmap, this falls back to regular file I/O.
|
||||
#[cfg(not(unix))]
|
||||
pub async fn from_file_read(file: &tokio::fs::File, offset: u64, size: usize) -> Result<Self, ZeroCopyReadError> {
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom};
|
||||
|
||||
let mut cloned = file.try_clone().await?;
|
||||
cloned.seek(SeekFrom::Start(offset)).await?;
|
||||
|
||||
let mut buffer = vec![0u8; size];
|
||||
cloned.read_exact(&mut buffer).await?;
|
||||
|
||||
Ok(Self {
|
||||
data: Bytes::from(buffer),
|
||||
pos: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Historical name for `from_file_read`.
|
||||
#[deprecated(
|
||||
since = "1.0.0-beta.8",
|
||||
note = "use from_file_read; this method performs normal reads into owned Bytes"
|
||||
)]
|
||||
pub async fn from_file_mmap(file: &tokio::fs::File, offset: u64, size: usize) -> Result<Self, ZeroCopyReadError> {
|
||||
Self::from_file_read(file, offset, size).await
|
||||
}
|
||||
|
||||
/// Get the remaining data as Bytes (zero-copy).
|
||||
///
|
||||
/// This returns a slice of the remaining data without copying.
|
||||
/// The returned Bytes shares the underlying memory with this reader.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let remaining = reader.remaining_bytes();
|
||||
/// println!("Remaining: {} bytes", remaining.len());
|
||||
/// ```
|
||||
pub fn remaining_bytes(&self) -> Bytes {
|
||||
self.data.slice(self.pos..)
|
||||
}
|
||||
|
||||
/// Get the total length of the data.
|
||||
pub fn len(&self) -> usize {
|
||||
self.data.len()
|
||||
}
|
||||
|
||||
/// Check if the reader has reached the end.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.pos >= self.data.len()
|
||||
}
|
||||
|
||||
/// Get the current read position.
|
||||
pub fn position(&self) -> usize {
|
||||
self.pos
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for BytesBufferedReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let remaining = self.data.len() - self.pos;
|
||||
if remaining == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
let to_read = std::cmp::min(remaining, buf.remaining());
|
||||
let slice = &self.data[self.pos..self.pos + to_read];
|
||||
buf.put_slice(slice);
|
||||
self.pos += to_read;
|
||||
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BytesBufferedReader {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("BytesBufferedReader")
|
||||
.field("data_len", &self.data.len())
|
||||
.field("pos", &self.pos)
|
||||
.field("remaining", &(self.data.len() - self.pos))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
fn temp_file_path(test_name: &str) -> PathBuf {
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system time should be after unix epoch")
|
||||
.as_nanos();
|
||||
std::env::temp_dir().join(format!("rustfs-io-core-{test_name}-{}-{nonce}", std::process::id()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_from_bytes() {
|
||||
let data = Bytes::from("hello world");
|
||||
let mut reader = BytesBufferedReader::from_bytes(data.clone());
|
||||
|
||||
let mut buf = [0u8; 11];
|
||||
let n = reader.read(&mut buf[..]).await.unwrap();
|
||||
|
||||
assert_eq!(n, 11);
|
||||
assert_eq!(&buf[..n], b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_preferred_reader_alias() {
|
||||
let data = Bytes::from("hello world");
|
||||
let mut reader = BytesBufferedReader::from_bytes(data);
|
||||
|
||||
let mut buf = [0u8; 5];
|
||||
let n = reader.read(&mut buf[..]).await.expect("read bytes from alias");
|
||||
|
||||
assert_eq!(n, 5);
|
||||
assert_eq!(&buf[..n], b"hello");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_from_file_read_reads_requested_range() {
|
||||
let path = temp_file_path("from-file-read");
|
||||
tokio::fs::write(&path, b"hello world")
|
||||
.await
|
||||
.expect("write temp file for reader test");
|
||||
|
||||
let file = tokio::fs::File::open(&path).await.expect("open temp file for reader test");
|
||||
let mut reader = BytesBufferedReader::from_file_read(&file, 6, 5)
|
||||
.await
|
||||
.expect("read requested range into Bytes");
|
||||
|
||||
let mut output = Vec::new();
|
||||
reader.read_to_end(&mut output).await.expect("drain reader output");
|
||||
|
||||
assert_eq!(output, b"world");
|
||||
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(deprecated)]
|
||||
async fn test_from_file_mmap_legacy_alias_reads_requested_range() {
|
||||
let path = temp_file_path("from-file-mmap");
|
||||
tokio::fs::write(&path, b"hello world")
|
||||
.await
|
||||
.expect("write temp file for legacy reader test");
|
||||
|
||||
let file = tokio::fs::File::open(&path)
|
||||
.await
|
||||
.expect("open temp file for legacy reader test");
|
||||
let mut reader = BytesBufferedReader::from_file_mmap(&file, 0, 5)
|
||||
.await
|
||||
.expect("read requested range through legacy alias");
|
||||
|
||||
let mut output = Vec::new();
|
||||
reader.read_to_end(&mut output).await.expect("drain legacy reader output");
|
||||
|
||||
assert_eq!(output, b"hello");
|
||||
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remaining_bytes() {
|
||||
let data = Bytes::from("hello world");
|
||||
let reader = BytesBufferedReader::from_bytes(data);
|
||||
|
||||
let remaining = reader.remaining_bytes();
|
||||
assert_eq!(remaining.len(), 11);
|
||||
assert_eq!(&remaining[..], b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_position() {
|
||||
let data = Bytes::from("hello world");
|
||||
let mut reader = BytesBufferedReader::from_bytes(data);
|
||||
|
||||
assert_eq!(reader.position(), 0);
|
||||
|
||||
let mut buf = [0u8; 5];
|
||||
reader.read_exact(&mut buf[..]).await.unwrap();
|
||||
|
||||
assert_eq!(reader.position(), 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_is_empty() {
|
||||
let data = Bytes::from("");
|
||||
let reader = BytesBufferedReader::from_bytes(data);
|
||||
assert!(reader.is_empty());
|
||||
|
||||
let data = Bytes::from("hello");
|
||||
let reader = BytesBufferedReader::from_bytes(data);
|
||||
assert!(!reader.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(deprecated)]
|
||||
async fn test_legacy_reader_alias() {
|
||||
let data = Bytes::from("hello world");
|
||||
let mut reader = ZeroCopyObjectReader::from_bytes(data);
|
||||
|
||||
let mut buf = [0u8; 5];
|
||||
let n = reader.read(&mut buf[..]).await.expect("read bytes through legacy alias");
|
||||
|
||||
assert_eq!(n, 5);
|
||||
assert_eq!(&buf[..n], b"hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,882 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! I/O scheduler for adaptive buffer sizing and load management.
|
||||
//!
|
||||
//! This module provides the core I/O scheduling logic that determines
|
||||
//! optimal buffer sizes, I/O strategies, and load management decisions.
|
||||
|
||||
use crate::config::IoSchedulerConfig;
|
||||
use crate::io_profile::{AccessPattern, StorageMedia, StorageProfile};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
/// I/O priority levels.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum IoPriority {
|
||||
/// High priority for small, latency-sensitive operations.
|
||||
High,
|
||||
/// Normal priority for standard operations.
|
||||
#[default]
|
||||
Normal,
|
||||
/// Low priority for large, throughput-oriented operations.
|
||||
Low,
|
||||
}
|
||||
|
||||
impl IoPriority {
|
||||
/// Determine priority based on request size.
|
||||
///
|
||||
/// A negative `size` means the size is unknown (-1 by convention) and maps
|
||||
/// to `Normal`; casting it to `usize` would wrap to a huge value and
|
||||
/// misclassify the request as `Low`.
|
||||
pub fn from_size(size: i64, high_threshold: usize, low_threshold: usize) -> Self {
|
||||
if size < 0 {
|
||||
return IoPriority::Normal;
|
||||
}
|
||||
let size = size as usize;
|
||||
if size < high_threshold {
|
||||
IoPriority::High
|
||||
} else if size > low_threshold {
|
||||
IoPriority::Low
|
||||
} else {
|
||||
IoPriority::Normal
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the priority as a string for metrics labels.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
IoPriority::High => "high",
|
||||
IoPriority::Normal => "normal",
|
||||
IoPriority::Low => "low",
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this is high priority.
|
||||
pub fn is_high(&self) -> bool {
|
||||
matches!(self, IoPriority::High)
|
||||
}
|
||||
|
||||
/// Check if this is normal priority.
|
||||
pub fn is_normal(&self) -> bool {
|
||||
matches!(self, IoPriority::Normal)
|
||||
}
|
||||
|
||||
/// Check if this is low priority.
|
||||
pub fn is_low(&self) -> bool {
|
||||
matches!(self, IoPriority::Low)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for IoPriority {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// I/O load level.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Default)]
|
||||
pub enum IoLoadLevel {
|
||||
/// Low load - system is underutilized.
|
||||
Low,
|
||||
/// Medium load - system is moderately utilized.
|
||||
#[default]
|
||||
Medium,
|
||||
/// High load - system is heavily utilized.
|
||||
High,
|
||||
/// Critical load - system is overloaded.
|
||||
Critical,
|
||||
}
|
||||
|
||||
impl IoLoadLevel {
|
||||
/// Get the load level as a string for metrics labels.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
IoLoadLevel::Low => "low",
|
||||
IoLoadLevel::Medium => "medium",
|
||||
IoLoadLevel::High => "high",
|
||||
IoLoadLevel::Critical => "critical",
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine load level from wait time.
|
||||
pub fn from_wait_time(wait_time: Duration, low_threshold: Duration, high_threshold: Duration) -> Self {
|
||||
if wait_time <= low_threshold {
|
||||
IoLoadLevel::Low
|
||||
} else if wait_time <= high_threshold {
|
||||
IoLoadLevel::Medium
|
||||
} else if wait_time <= high_threshold * 2 {
|
||||
IoLoadLevel::High
|
||||
} else {
|
||||
IoLoadLevel::Critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for IoLoadLevel {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bandwidth tier classification.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum BandwidthTier {
|
||||
/// Low bandwidth (< 100 MB/s).
|
||||
Low,
|
||||
/// Medium bandwidth (100-500 MB/s).
|
||||
#[default]
|
||||
Medium,
|
||||
/// High bandwidth (> 500 MB/s).
|
||||
High,
|
||||
/// Unknown bandwidth.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl BandwidthTier {
|
||||
/// Determine bandwidth tier from bytes per second.
|
||||
pub fn from_bps(bps: u64) -> Self {
|
||||
const MB: u64 = 1024 * 1024;
|
||||
if bps < 100 * MB {
|
||||
BandwidthTier::Low
|
||||
} else if bps < 500 * MB {
|
||||
BandwidthTier::Medium
|
||||
} else {
|
||||
BandwidthTier::High
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the tier as a string for metrics labels.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
BandwidthTier::Low => "low",
|
||||
BandwidthTier::Medium => "medium",
|
||||
BandwidthTier::High => "high",
|
||||
BandwidthTier::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// I/O strategy decision.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IoStrategy {
|
||||
/// Buffer size to use for I/O operations.
|
||||
pub buffer_size: usize,
|
||||
/// Buffer multiplier based on storage media.
|
||||
pub buffer_multiplier: f64,
|
||||
/// Whether to enable readahead.
|
||||
pub enable_readahead: bool,
|
||||
/// Whether to use buffered I/O.
|
||||
pub use_buffered_io: bool,
|
||||
|
||||
// Performance state
|
||||
/// Current number of concurrent requests.
|
||||
pub concurrent_requests: usize,
|
||||
/// Observed bandwidth in bytes per second.
|
||||
pub observed_bandwidth_bps: Option<u64>,
|
||||
/// Bandwidth tier classification.
|
||||
pub bandwidth_tier: BandwidthTier,
|
||||
/// Current load level.
|
||||
pub load_level: IoLoadLevel,
|
||||
|
||||
// Priority
|
||||
/// I/O priority for this operation.
|
||||
pub priority: IoPriority,
|
||||
|
||||
// Decision flags
|
||||
/// Whether to throttle random I/O.
|
||||
pub should_throttle_random_io: bool,
|
||||
/// Whether to expand buffer for sequential access.
|
||||
pub should_expand_for_sequential: bool,
|
||||
/// Whether to reduce buffer due to concurrency.
|
||||
pub should_reduce_for_concurrency: bool,
|
||||
/// Whether to reduce buffer due to low bandwidth.
|
||||
pub should_reduce_for_bandwidth: bool,
|
||||
}
|
||||
|
||||
impl Default for IoStrategy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
buffer_size: 128 * 1024,
|
||||
buffer_multiplier: 1.0,
|
||||
enable_readahead: true,
|
||||
use_buffered_io: true,
|
||||
concurrent_requests: 0,
|
||||
observed_bandwidth_bps: None,
|
||||
bandwidth_tier: BandwidthTier::Medium,
|
||||
load_level: IoLoadLevel::Low,
|
||||
priority: IoPriority::Normal,
|
||||
should_throttle_random_io: false,
|
||||
should_expand_for_sequential: false,
|
||||
should_reduce_for_concurrency: false,
|
||||
should_reduce_for_bandwidth: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IoStrategy {
|
||||
/// Create a new strategy with default values.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Create a strategy for sequential access.
|
||||
pub fn sequential(buffer_size: usize) -> Self {
|
||||
Self {
|
||||
buffer_size,
|
||||
enable_readahead: true,
|
||||
should_expand_for_sequential: true,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a strategy for random access.
|
||||
pub fn random(buffer_size: usize) -> Self {
|
||||
Self {
|
||||
buffer_size,
|
||||
enable_readahead: false,
|
||||
should_throttle_random_io: true,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// I/O load metrics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct IoLoadMetrics {
|
||||
/// Number of samples in the current window.
|
||||
pub sample_count: usize,
|
||||
/// Total wait time in the window.
|
||||
pub total_wait_time: Duration,
|
||||
/// Maximum wait time in the window.
|
||||
pub max_wait_time: Duration,
|
||||
/// Average wait time.
|
||||
pub avg_wait_time: Duration,
|
||||
/// Current load level.
|
||||
pub load_level: IoLoadLevel,
|
||||
}
|
||||
|
||||
impl IoLoadMetrics {
|
||||
/// Create new load metrics.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Add a wait time sample.
|
||||
pub fn add_sample(&mut self, wait_time: Duration) {
|
||||
self.sample_count += 1;
|
||||
self.total_wait_time += wait_time;
|
||||
if wait_time > self.max_wait_time {
|
||||
self.max_wait_time = wait_time;
|
||||
}
|
||||
self.avg_wait_time = if self.sample_count > 0 {
|
||||
self.total_wait_time / self.sample_count as u32
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
}
|
||||
|
||||
/// Update load level based on thresholds.
|
||||
pub fn update_load_level(&mut self, low_threshold: Duration, high_threshold: Duration) {
|
||||
self.load_level = IoLoadLevel::from_wait_time(self.avg_wait_time, low_threshold, high_threshold);
|
||||
}
|
||||
|
||||
/// Reset the metrics.
|
||||
pub fn reset(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
}
|
||||
|
||||
/// I/O scheduler.
|
||||
pub struct IoScheduler {
|
||||
/// Scheduler configuration.
|
||||
config: IoSchedulerConfig,
|
||||
/// Active request counter.
|
||||
active_requests: AtomicUsize,
|
||||
/// Load metrics.
|
||||
load_metrics: std::sync::Mutex<IoLoadMetrics>,
|
||||
}
|
||||
|
||||
impl IoScheduler {
|
||||
/// Create a new I/O scheduler with the given configuration.
|
||||
pub fn new(config: IoSchedulerConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
active_requests: AtomicUsize::new(0),
|
||||
load_metrics: std::sync::Mutex::new(IoLoadMetrics::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new I/O scheduler with default configuration.
|
||||
pub fn with_defaults() -> Self {
|
||||
Self::new(IoSchedulerConfig::default())
|
||||
}
|
||||
|
||||
/// Get the scheduler configuration.
|
||||
pub fn config(&self) -> &IoSchedulerConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Get the current number of active requests.
|
||||
pub fn active_requests(&self) -> usize {
|
||||
self.active_requests.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Increment the active request count.
|
||||
pub fn increment_requests(&self) {
|
||||
self.active_requests.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Decrement the active request count.
|
||||
pub fn decrement_requests(&self) {
|
||||
self.active_requests.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Calculate I/O strategy for a request.
|
||||
pub fn calculate_strategy(&self, file_size: i64, permit_wait_time: Duration, is_sequential: bool) -> IoStrategy {
|
||||
let concurrent_requests = self.active_requests.load(Ordering::Relaxed);
|
||||
|
||||
// Determine priority based on file size
|
||||
let priority = IoPriority::from_size(
|
||||
file_size,
|
||||
self.config.high_priority_size_threshold,
|
||||
self.config.low_priority_size_threshold,
|
||||
);
|
||||
|
||||
// Determine load level
|
||||
let load_level =
|
||||
IoLoadLevel::from_wait_time(permit_wait_time, self.config.load_low_threshold(), self.config.load_high_threshold());
|
||||
|
||||
// Calculate base buffer size
|
||||
let base_buffer = self.config.base_buffer_size;
|
||||
|
||||
// Adjust for concurrency
|
||||
let concurrency_factor = match concurrent_requests {
|
||||
0..=2 => 1.0,
|
||||
3..=4 => 0.75,
|
||||
5..=8 => 0.5,
|
||||
_ => 0.4,
|
||||
};
|
||||
|
||||
// Adjust for load level
|
||||
let load_factor = match load_level {
|
||||
IoLoadLevel::Low => 1.2,
|
||||
IoLoadLevel::Medium => 1.0,
|
||||
IoLoadLevel::High => 0.7,
|
||||
IoLoadLevel::Critical => 0.5,
|
||||
};
|
||||
|
||||
// Adjust for access pattern
|
||||
let sequential_factor = if is_sequential { 1.5 } else { 1.0 };
|
||||
|
||||
// Calculate final buffer size
|
||||
let buffer_size = (base_buffer as f64 * concurrency_factor * load_factor * sequential_factor) as usize;
|
||||
let buffer_size = buffer_size.clamp(self.config.min_buffer_size, self.config.max_buffer_size);
|
||||
|
||||
IoStrategy {
|
||||
buffer_size,
|
||||
buffer_multiplier: concurrency_factor * load_factor * sequential_factor,
|
||||
enable_readahead: is_sequential && load_level != IoLoadLevel::Critical,
|
||||
use_buffered_io: true,
|
||||
concurrent_requests,
|
||||
observed_bandwidth_bps: None,
|
||||
bandwidth_tier: BandwidthTier::Unknown,
|
||||
load_level,
|
||||
priority,
|
||||
should_throttle_random_io: !is_sequential && load_level >= IoLoadLevel::High,
|
||||
should_expand_for_sequential: is_sequential && load_level <= IoLoadLevel::Medium,
|
||||
should_reduce_for_concurrency: concurrent_requests > 4,
|
||||
should_reduce_for_bandwidth: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate multi-factor I/O strategy.
|
||||
pub fn calculate_multi_factor_strategy(
|
||||
&self,
|
||||
file_size: i64,
|
||||
permit_wait_time: Duration,
|
||||
is_sequential: bool,
|
||||
storage_profile: Option<&StorageProfile>,
|
||||
) -> IoStrategy {
|
||||
let mut strategy = self.calculate_strategy(file_size, permit_wait_time, is_sequential);
|
||||
|
||||
// Apply storage profile adjustments
|
||||
if let Some(profile) = storage_profile {
|
||||
// Adjust buffer size based on storage media
|
||||
let media_factor = match profile.media {
|
||||
StorageMedia::Nvme => 1.5,
|
||||
StorageMedia::Ssd => 1.2,
|
||||
StorageMedia::Hdd => 0.8,
|
||||
StorageMedia::Unknown => 1.0,
|
||||
};
|
||||
|
||||
strategy.buffer_size = (strategy.buffer_size as f64 * media_factor).min(self.config.max_buffer_size as f64) as usize;
|
||||
|
||||
// Apply sequential boost if applicable
|
||||
if is_sequential {
|
||||
strategy.buffer_size = (strategy.buffer_size as f64 * profile.sequential_boost_multiplier)
|
||||
.min(self.config.max_buffer_size as f64) as usize;
|
||||
}
|
||||
|
||||
// Apply random penalty if applicable
|
||||
if !is_sequential {
|
||||
strategy.buffer_size = (strategy.buffer_size as f64 * profile.random_penalty_multiplier)
|
||||
.max(self.config.min_buffer_size as f64) as usize;
|
||||
}
|
||||
|
||||
// Update readahead preference
|
||||
strategy.enable_readahead = strategy.enable_readahead && profile.prefers_readahead;
|
||||
}
|
||||
|
||||
strategy
|
||||
}
|
||||
|
||||
/// Record a wait time sample for load tracking.
|
||||
pub fn record_wait_time(&self, wait_time: Duration) {
|
||||
if let Ok(mut metrics) = self.load_metrics.lock() {
|
||||
metrics.add_sample(wait_time);
|
||||
metrics.update_load_level(self.config.load_low_threshold(), self.config.load_high_threshold());
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current load metrics.
|
||||
pub fn load_metrics(&self) -> IoLoadMetrics {
|
||||
if let Ok(metrics) = self.load_metrics.lock() {
|
||||
metrics.clone()
|
||||
} else {
|
||||
IoLoadMetrics::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for IoScheduler {
|
||||
fn default() -> Self {
|
||||
Self::with_defaults()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Buffer Size Calculation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Constants for buffer size calculations.
|
||||
pub const KI_B: usize = 1024;
|
||||
pub const MI_B: usize = 1024 * 1024;
|
||||
|
||||
/// Get concurrency-aware buffer size.
|
||||
///
|
||||
/// Adjusts buffer size based on the current level of concurrent requests.
|
||||
/// Higher concurrency leads to smaller buffers to reduce memory pressure.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file_size` - Size of the file being read (-1 if unknown)
|
||||
/// * `base_buffer_size` - Base buffer size from workload profile
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Adjusted buffer size in bytes
|
||||
pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize {
|
||||
// Get current concurrency level from global counter
|
||||
let concurrent_requests = 1; // Default to 1 if no global counter available
|
||||
|
||||
// Define concurrency thresholds
|
||||
let medium_threshold = 4;
|
||||
let high_threshold = 8;
|
||||
|
||||
// Calculate adaptive multiplier based on concurrency
|
||||
let adaptive_multiplier = if concurrent_requests <= 2 {
|
||||
// Low concurrency (1-2): use full buffer size
|
||||
1.0
|
||||
} else if concurrent_requests <= medium_threshold {
|
||||
// Medium concurrency (3-4): slightly reduce buffer size (75% of base)
|
||||
0.75
|
||||
} else if concurrent_requests <= high_threshold {
|
||||
// Higher concurrency (5-8): more aggressive reduction (50% of base)
|
||||
0.5
|
||||
} else {
|
||||
// Very high concurrency (>8): minimize memory per request (40% of base)
|
||||
0.4
|
||||
};
|
||||
|
||||
// Calculate the adjusted buffer size
|
||||
let adjusted_size = (base_buffer_size as f64 * adaptive_multiplier) as usize;
|
||||
|
||||
// Ensure we stay within reasonable bounds
|
||||
let min_buffer = if file_size > 0 && file_size < 100 * KI_B as i64 {
|
||||
32 * KI_B // For very small files, use minimum buffer
|
||||
} else {
|
||||
64 * KI_B // Standard minimum buffer size
|
||||
};
|
||||
|
||||
let max_buffer = if concurrent_requests > high_threshold {
|
||||
256 * KI_B // Cap at 256KB for high concurrency
|
||||
} else {
|
||||
MI_B // Cap at 1MB for lower concurrency
|
||||
};
|
||||
|
||||
adjusted_size.clamp(min_buffer, max_buffer)
|
||||
}
|
||||
|
||||
/// Advanced concurrency-aware buffer sizing with file size optimization.
|
||||
///
|
||||
/// This enhanced version considers both concurrency level and file size patterns
|
||||
/// to provide even better performance characteristics.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file_size` - Size of the file being read (-1 if unknown)
|
||||
/// * `base_buffer_size` - Baseline buffer size from workload profile
|
||||
/// * `is_sequential` - Whether this is a sequential read (hint for optimization)
|
||||
/// * `concurrent_requests` - Current number of concurrent requests
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Optimized buffer size in bytes
|
||||
pub fn get_advanced_buffer_size(
|
||||
file_size: i64,
|
||||
base_buffer_size: usize,
|
||||
is_sequential: bool,
|
||||
concurrent_requests: usize,
|
||||
) -> usize {
|
||||
// For very small files, use smaller buffers regardless of concurrency
|
||||
if file_size > 0 && file_size < 256 * KI_B as i64 {
|
||||
return (file_size as usize / 4).clamp(16 * KI_B, 64 * KI_B);
|
||||
}
|
||||
|
||||
// Base calculation from standard function
|
||||
let standard_size = get_concurrency_aware_buffer_size(file_size, base_buffer_size);
|
||||
|
||||
let medium_threshold = 4;
|
||||
let high_threshold = 8;
|
||||
|
||||
// For sequential reads, we can be more aggressive with buffer sizes
|
||||
if is_sequential && concurrent_requests <= medium_threshold {
|
||||
// Boost buffer size for sequential reads under low concurrency
|
||||
let boosted = (standard_size as f64 * 1.5) as usize;
|
||||
return boosted.min(MI_B);
|
||||
}
|
||||
|
||||
// For random reads under high concurrency, reduce buffer size
|
||||
if !is_sequential && concurrent_requests > high_threshold {
|
||||
let reduced = (standard_size as f64 * 0.7) as usize;
|
||||
return reduced.max(32 * KI_B);
|
||||
}
|
||||
|
||||
standard_size
|
||||
}
|
||||
|
||||
/// Get buffer size with storage media optimization.
|
||||
///
|
||||
/// Adjusts buffer size based on storage media characteristics.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `base_size` - Base buffer size
|
||||
/// * `media` - Storage media type
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Optimized buffer size for the storage media
|
||||
pub fn get_buffer_size_for_media(base_size: usize, media: StorageMedia) -> usize {
|
||||
let multiplier = match media {
|
||||
StorageMedia::Nvme => 1.5, // NVMe can handle larger buffers
|
||||
StorageMedia::Ssd => 1.2, // SSD benefits from moderate buffers
|
||||
StorageMedia::Hdd => 0.8, // HDD prefers smaller buffers to reduce seek overhead
|
||||
StorageMedia::Unknown => 1.0,
|
||||
};
|
||||
|
||||
(base_size as f64 * multiplier).min(MI_B as f64) as usize
|
||||
}
|
||||
|
||||
/// Calculate optimal buffer size using multi-factor analysis.
|
||||
///
|
||||
/// This is the main entry point for buffer size calculation, considering
|
||||
/// all factors: concurrency, storage media, access pattern, and load.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file_size` - Size of the file being read
|
||||
/// * `base_buffer_size` - Base buffer size
|
||||
/// * `is_sequential` - Whether access is sequential
|
||||
/// * `concurrent_requests` - Current concurrency level
|
||||
/// * `media` - Storage media type
|
||||
/// * `load_level` - Current I/O load level
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Optimally calculated buffer size
|
||||
pub fn calculate_optimal_buffer_size(
|
||||
file_size: i64,
|
||||
base_buffer_size: usize,
|
||||
is_sequential: bool,
|
||||
concurrent_requests: usize,
|
||||
media: StorageMedia,
|
||||
load_level: IoLoadLevel,
|
||||
) -> usize {
|
||||
// Start with advanced buffer size calculation
|
||||
let mut buffer_size = get_advanced_buffer_size(file_size, base_buffer_size, is_sequential, concurrent_requests);
|
||||
|
||||
// Apply storage media optimization
|
||||
buffer_size = get_buffer_size_for_media(buffer_size, media);
|
||||
|
||||
// Apply load-based adjustment
|
||||
let load_multiplier = match load_level {
|
||||
IoLoadLevel::Low => 1.2,
|
||||
IoLoadLevel::Medium => 1.0,
|
||||
IoLoadLevel::High => 0.7,
|
||||
IoLoadLevel::Critical => 0.5,
|
||||
};
|
||||
|
||||
buffer_size = (buffer_size as f64 * load_multiplier) as usize;
|
||||
|
||||
// Final bounds check
|
||||
buffer_size.clamp(32 * KI_B, MI_B)
|
||||
}
|
||||
|
||||
/// I/O scheduling context for multi-factor strategy calculation.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IoSchedulingContext {
|
||||
/// File size in bytes (-1 if unknown).
|
||||
pub file_size: i64,
|
||||
/// Base buffer size from configuration.
|
||||
pub base_buffer_size: usize,
|
||||
/// Time spent waiting for permit.
|
||||
pub permit_wait_duration: Duration,
|
||||
/// Whether access is sequential.
|
||||
pub is_sequential_hint: bool,
|
||||
/// Detected access pattern.
|
||||
pub access_pattern: AccessPattern,
|
||||
/// Detected storage media.
|
||||
pub storage_media: StorageMedia,
|
||||
/// Observed bandwidth in bytes per second.
|
||||
pub observed_bandwidth_bps: Option<u64>,
|
||||
/// Current concurrent request count.
|
||||
pub concurrent_requests: usize,
|
||||
}
|
||||
|
||||
impl Default for IoSchedulingContext {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
file_size: -1,
|
||||
base_buffer_size: 128 * KI_B,
|
||||
permit_wait_duration: Duration::ZERO,
|
||||
is_sequential_hint: true,
|
||||
access_pattern: AccessPattern::Unknown,
|
||||
storage_media: StorageMedia::Unknown,
|
||||
observed_bandwidth_bps: None,
|
||||
concurrent_requests: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IoSchedulingContext {
|
||||
/// Create a new scheduling context.
|
||||
pub fn new(file_size: i64, base_buffer_size: usize) -> Self {
|
||||
Self {
|
||||
file_size,
|
||||
base_buffer_size,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder pattern: set sequential hint.
|
||||
pub fn with_sequential(mut self, is_sequential: bool) -> Self {
|
||||
self.is_sequential_hint = is_sequential;
|
||||
self.access_pattern = if is_sequential {
|
||||
AccessPattern::Sequential
|
||||
} else {
|
||||
AccessPattern::Random
|
||||
};
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder pattern: set storage media.
|
||||
pub fn with_media(mut self, media: StorageMedia) -> Self {
|
||||
self.storage_media = media;
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder pattern: set bandwidth.
|
||||
pub fn with_bandwidth(mut self, bps: u64) -> Self {
|
||||
self.observed_bandwidth_bps = Some(bps);
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder pattern: set concurrency.
|
||||
pub fn with_concurrency(mut self, count: usize) -> Self {
|
||||
self.concurrent_requests = count;
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder pattern: set wait duration.
|
||||
pub fn with_wait_duration(mut self, duration: Duration) -> Self {
|
||||
self.permit_wait_duration = duration;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_io_priority() {
|
||||
assert_eq!(IoPriority::from_size(1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::High);
|
||||
assert_eq!(IoPriority::from_size(1024 * 1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal);
|
||||
assert_eq!(IoPriority::from_size(10 * 1024 * 1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::Low);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_priority_unknown_size_is_normal() {
|
||||
// -1 means "size unknown" and must not wrap to usize::MAX (=> Low).
|
||||
assert_eq!(IoPriority::from_size(-1, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal);
|
||||
assert_eq!(IoPriority::from_size(i64::MIN, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_load_level() {
|
||||
let low = Duration::from_millis(5);
|
||||
let high = Duration::from_millis(50);
|
||||
|
||||
assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(1), low, high), IoLoadLevel::Low);
|
||||
assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(20), low, high), IoLoadLevel::Medium);
|
||||
assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(60), low, high), IoLoadLevel::High);
|
||||
assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(150), low, high), IoLoadLevel::Critical);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bandwidth_tier() {
|
||||
assert_eq!(BandwidthTier::from_bps(50 * 1024 * 1024), BandwidthTier::Low);
|
||||
assert_eq!(BandwidthTier::from_bps(200 * 1024 * 1024), BandwidthTier::Medium);
|
||||
assert_eq!(BandwidthTier::from_bps(600 * 1024 * 1024), BandwidthTier::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_strategy_default() {
|
||||
let strategy = IoStrategy::default();
|
||||
assert!(strategy.buffer_size > 0);
|
||||
assert!(strategy.enable_readahead);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_scheduler() {
|
||||
let scheduler = IoScheduler::with_defaults();
|
||||
|
||||
let strategy = scheduler.calculate_strategy(1024 * 1024, Duration::from_millis(5), true);
|
||||
assert!(strategy.buffer_size > 0);
|
||||
assert!(strategy.enable_readahead);
|
||||
assert_eq!(strategy.load_level, IoLoadLevel::Low);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_scheduler_with_concurrency() {
|
||||
let scheduler = IoScheduler::with_defaults();
|
||||
|
||||
// Simulate concurrent requests
|
||||
scheduler.increment_requests();
|
||||
scheduler.increment_requests();
|
||||
scheduler.increment_requests();
|
||||
|
||||
let strategy = scheduler.calculate_strategy(1024 * 1024, Duration::from_millis(5), true);
|
||||
assert_eq!(strategy.concurrent_requests, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_metrics() {
|
||||
let mut metrics = IoLoadMetrics::new();
|
||||
|
||||
metrics.add_sample(Duration::from_millis(10));
|
||||
metrics.add_sample(Duration::from_millis(20));
|
||||
metrics.add_sample(Duration::from_millis(30));
|
||||
|
||||
assert_eq!(metrics.sample_count, 3);
|
||||
assert_eq!(metrics.avg_wait_time, Duration::from_millis(20));
|
||||
assert_eq!(metrics.max_wait_time, Duration::from_millis(30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_concurrency_aware_buffer_size() {
|
||||
// Test with default concurrency (1)
|
||||
let size = get_concurrency_aware_buffer_size(1024 * 1024, 128 * KI_B);
|
||||
assert!(size >= 64 * KI_B);
|
||||
assert!(size <= MI_B);
|
||||
|
||||
// Test with small file
|
||||
let size = get_concurrency_aware_buffer_size(50 * KI_B as i64, 128 * KI_B);
|
||||
assert!(size >= 32 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_advanced_buffer_size() {
|
||||
// Sequential read with low concurrency
|
||||
let size = get_advanced_buffer_size(10 * MI_B as i64, 128 * KI_B, true, 2);
|
||||
assert!(size >= 128 * KI_B);
|
||||
|
||||
// Random read with high concurrency
|
||||
let size = get_advanced_buffer_size(10 * MI_B as i64, 128 * KI_B, false, 10);
|
||||
assert!(size >= 32 * KI_B);
|
||||
|
||||
// Very small file
|
||||
let size = get_advanced_buffer_size(100 * KI_B as i64, 128 * KI_B, true, 1);
|
||||
assert!(size <= 64 * KI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_buffer_size_for_media() {
|
||||
let base = 128 * KI_B;
|
||||
|
||||
// NVMe should get larger buffers
|
||||
let nvme_size = get_buffer_size_for_media(base, StorageMedia::Nvme);
|
||||
assert!(nvme_size > base);
|
||||
|
||||
// SSD should get slightly larger buffers
|
||||
let ssd_size = get_buffer_size_for_media(base, StorageMedia::Ssd);
|
||||
assert!(ssd_size > base);
|
||||
|
||||
// HDD should get smaller buffers
|
||||
let hdd_size = get_buffer_size_for_media(base, StorageMedia::Hdd);
|
||||
assert!(hdd_size < base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_calculate_optimal_buffer_size() {
|
||||
// Low load, sequential, NVMe
|
||||
let size = calculate_optimal_buffer_size(10 * MI_B as i64, 128 * KI_B, true, 2, StorageMedia::Nvme, IoLoadLevel::Low);
|
||||
assert!(size >= 32 * KI_B);
|
||||
assert!(size <= MI_B);
|
||||
|
||||
// Critical load, random, HDD
|
||||
let size =
|
||||
calculate_optimal_buffer_size(10 * MI_B as i64, 128 * KI_B, false, 10, StorageMedia::Hdd, IoLoadLevel::Critical);
|
||||
assert!(size >= 32 * KI_B);
|
||||
assert!(size <= MI_B);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_scheduling_context() {
|
||||
let ctx = IoSchedulingContext::new(10 * MI_B as i64, 256 * KI_B)
|
||||
.with_sequential(true)
|
||||
.with_media(StorageMedia::Nvme)
|
||||
.with_bandwidth(500 * MI_B as u64)
|
||||
.with_concurrency(4);
|
||||
|
||||
assert_eq!(ctx.file_size, 10 * MI_B as i64);
|
||||
assert_eq!(ctx.base_buffer_size, 256 * KI_B);
|
||||
assert!(ctx.is_sequential_hint);
|
||||
assert_eq!(ctx.storage_media, StorageMedia::Nvme);
|
||||
assert_eq!(ctx.observed_bandwidth_bps, Some(500 * MI_B as u64));
|
||||
assert_eq!(ctx.concurrent_requests, 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared memory pool for zero-copy data sharing.
|
||||
//!
|
||||
//! This module provides Arc-based shared memory management for
|
||||
//! efficient cross-task data passing without serialization.
|
||||
|
||||
use std::convert::AsRef;
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Shared memory pool configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SharedMemoryConfig {
|
||||
/// Whether shared memory is enabled
|
||||
pub enabled: bool,
|
||||
|
||||
/// Maximum pool size in bytes
|
||||
pub max_pool_size: usize,
|
||||
|
||||
/// Maximum object size in bytes
|
||||
pub max_object_size: usize,
|
||||
}
|
||||
|
||||
impl Default for SharedMemoryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
max_pool_size: 100 * 1024 * 1024, // 100MB
|
||||
max_object_size: 10 * 1024 * 1024, // 10MB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared memory pool statistics.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SharedMemoryStats {
|
||||
/// Total number of objects created
|
||||
pub total_objects: AtomicU64,
|
||||
|
||||
/// Total number of shared references
|
||||
pub total_shared_refs: AtomicU64,
|
||||
|
||||
/// Current memory usage in bytes
|
||||
pub current_memory: AtomicU64,
|
||||
|
||||
/// Peak memory usage in bytes
|
||||
pub peak_memory: AtomicU64,
|
||||
}
|
||||
|
||||
/// Arc data metadata.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ArcMetadata {
|
||||
/// Size of the data (if measurable)
|
||||
pub size: Option<usize>,
|
||||
|
||||
/// Creation timestamp
|
||||
pub created_at: Instant,
|
||||
}
|
||||
|
||||
/// Arc-based data wrapper for zero-copy sharing.
|
||||
///
|
||||
/// This wrapper uses Arc to enable shared ownership of data
|
||||
/// across multiple tasks without copying.
|
||||
pub struct ArcData<T> {
|
||||
/// The wrapped data
|
||||
inner: Arc<T>,
|
||||
|
||||
/// Metadata about the data
|
||||
metadata: ArcMetadata,
|
||||
}
|
||||
|
||||
impl<T> Clone for ArcData<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
inner: Arc::clone(&self.inner),
|
||||
metadata: self.metadata.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ArcData<T> {
|
||||
/// Create a new ArcData wrapper.
|
||||
pub fn new(data: T) -> Self {
|
||||
ArcData {
|
||||
inner: Arc::new(data),
|
||||
metadata: ArcMetadata {
|
||||
size: None,
|
||||
created_at: Instant::now(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ArcData wrapper with known size.
|
||||
pub fn with_size(data: T, size: usize) -> Self {
|
||||
ArcData {
|
||||
inner: Arc::new(data),
|
||||
metadata: ArcMetadata {
|
||||
size: Some(size),
|
||||
created_at: Instant::now(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the reference count.
|
||||
pub fn ref_count(&self) -> usize {
|
||||
Arc::strong_count(&self.inner)
|
||||
}
|
||||
|
||||
/// Convert into the underlying Arc.
|
||||
pub fn into_arc(self) -> Arc<T> {
|
||||
self.inner
|
||||
}
|
||||
|
||||
/// Get the metadata.
|
||||
pub fn metadata(&self) -> &ArcMetadata {
|
||||
&self.metadata
|
||||
}
|
||||
|
||||
/// Get the size if known.
|
||||
pub fn size(&self) -> Option<usize> {
|
||||
self.metadata.size
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AsRef<T> for ArcData<T> {
|
||||
fn as_ref(&self) -> &T {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Deref for ArcData<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::fmt::Debug for ArcData<T> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ArcData")
|
||||
.field("ref_count", &self.ref_count())
|
||||
.field("metadata", &self.metadata)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared memory pool for managing Arc-based shared data.
|
||||
pub struct SharedMemoryPool {
|
||||
config: SharedMemoryConfig,
|
||||
stats: SharedMemoryStats,
|
||||
}
|
||||
|
||||
impl SharedMemoryPool {
|
||||
/// Create a new shared memory pool with the given configuration.
|
||||
pub fn new(config: SharedMemoryConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
stats: SharedMemoryStats::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new shared memory pool with default configuration.
|
||||
pub fn with_defaults() -> Self {
|
||||
Self::new(SharedMemoryConfig::default())
|
||||
}
|
||||
|
||||
/// Create shared data.
|
||||
///
|
||||
/// This method wraps the data in an ArcData for zero-copy sharing.
|
||||
pub fn create<T>(&self, data: T) -> ArcData<T> {
|
||||
self.stats.total_objects.fetch_add(1, Ordering::Relaxed);
|
||||
ArcData::new(data)
|
||||
}
|
||||
|
||||
/// Create shared data with known size.
|
||||
///
|
||||
/// This method tracks memory usage for statistics.
|
||||
pub fn create_with_size<T>(&self, data: T, size: usize) -> ArcData<T> {
|
||||
self.stats.total_objects.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
// Update memory statistics
|
||||
self.stats.current_memory.fetch_add(size as u64, Ordering::Relaxed);
|
||||
|
||||
// Update peak memory
|
||||
let current = self.stats.current_memory.load(Ordering::Relaxed);
|
||||
let mut peak = self.stats.peak_memory.load(Ordering::Relaxed);
|
||||
if current > peak {
|
||||
peak = current;
|
||||
self.stats.peak_memory.store(peak, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
ArcData::with_size(data, size)
|
||||
}
|
||||
|
||||
/// Share data by increasing reference count.
|
||||
///
|
||||
/// This method creates a new ArcData that shares the underlying data
|
||||
/// without copying.
|
||||
pub fn share<T>(&self, data: &ArcData<T>) -> ArcData<T> {
|
||||
self.stats.total_shared_refs.fetch_add(1, Ordering::Relaxed);
|
||||
data.clone()
|
||||
}
|
||||
|
||||
/// Get the statistics for this pool.
|
||||
pub fn stats(&self) -> &SharedMemoryStats {
|
||||
&self.stats
|
||||
}
|
||||
|
||||
/// Get the configuration for this pool.
|
||||
pub fn config(&self) -> &SharedMemoryConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Check if the pool is enabled.
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.config.enabled
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_arc_data_new() {
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
let arc_data = ArcData::new(data.clone());
|
||||
|
||||
assert_eq!(arc_data.as_ref(), &data);
|
||||
assert_eq!(arc_data.ref_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_arc_data_clone() {
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
let arc_data = ArcData::new(data);
|
||||
|
||||
assert_eq!(arc_data.ref_count(), 1);
|
||||
|
||||
let arc_data2 = arc_data.clone();
|
||||
assert_eq!(arc_data.ref_count(), 2);
|
||||
assert_eq!(arc_data2.ref_count(), 2);
|
||||
|
||||
let arc_data3 = arc_data.clone();
|
||||
assert_eq!(arc_data.ref_count(), 3);
|
||||
assert_eq!(arc_data2.ref_count(), 3);
|
||||
assert_eq!(arc_data3.ref_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_arc_data_deref() {
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
let arc_data = ArcData::new(data);
|
||||
|
||||
// Test Deref trait
|
||||
assert_eq!(arc_data.len(), 5);
|
||||
assert_eq!(arc_data[0], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shared_memory_pool_create() {
|
||||
let pool = SharedMemoryPool::with_defaults();
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
|
||||
let arc_data = pool.create(data.clone());
|
||||
|
||||
assert_eq!(arc_data.as_ref(), &data);
|
||||
assert_eq!(pool.stats().total_objects.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shared_memory_pool_share() {
|
||||
let pool = SharedMemoryPool::with_defaults();
|
||||
let data = vec![1u8, 2, 3, 4, 5];
|
||||
|
||||
let arc_data = pool.create(data);
|
||||
assert_eq!(arc_data.ref_count(), 1);
|
||||
|
||||
let shared = pool.share(&arc_data);
|
||||
assert_eq!(arc_data.ref_count(), 2);
|
||||
assert_eq!(shared.ref_count(), 2);
|
||||
assert_eq!(pool.stats().total_shared_refs.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shared_memory_pool_with_size() {
|
||||
let pool = SharedMemoryPool::with_defaults();
|
||||
let data = vec![1u8; 1024];
|
||||
|
||||
let arc_data = pool.create_with_size(data, 1024);
|
||||
|
||||
assert_eq!(arc_data.size(), Some(1024));
|
||||
assert_eq!(pool.stats().current_memory.load(Ordering::Relaxed), 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_config() {
|
||||
let config = SharedMemoryConfig::default();
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.max_pool_size, 100 * 1024 * 1024);
|
||||
assert_eq!(config.max_object_size, 10 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Timeout wrapper for I/O operations.
|
||||
//!
|
||||
//! This module provides timeout management for I/O operations with
|
||||
//! dynamic timeout calculation based on operation size.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Timeout configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TimeoutConfig {
|
||||
/// Base timeout for small operations.
|
||||
pub base_timeout: Duration,
|
||||
/// Timeout per MB of data.
|
||||
pub timeout_per_mb: Duration,
|
||||
/// Maximum timeout.
|
||||
pub max_timeout: Duration,
|
||||
/// Minimum timeout.
|
||||
pub min_timeout: Duration,
|
||||
/// GetObject operation timeout.
|
||||
pub get_object_timeout: Duration,
|
||||
/// PutObject operation timeout.
|
||||
pub put_object_timeout: Duration,
|
||||
/// ListObjects operation timeout.
|
||||
pub list_objects_timeout: Duration,
|
||||
/// Whether dynamic timeout is enabled.
|
||||
pub enable_dynamic_timeout: bool,
|
||||
}
|
||||
|
||||
impl Default for TimeoutConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base_timeout: Duration::from_secs(5),
|
||||
timeout_per_mb: Duration::from_millis(100),
|
||||
max_timeout: Duration::from_secs(300),
|
||||
min_timeout: Duration::from_secs(1),
|
||||
get_object_timeout: Duration::from_secs(30),
|
||||
put_object_timeout: Duration::from_secs(60),
|
||||
list_objects_timeout: Duration::from_secs(10),
|
||||
enable_dynamic_timeout: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TimeoutConfig {
|
||||
/// Create new timeout configuration.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Calculate dynamic timeout based on size.
|
||||
pub fn calculate_timeout(&self, size_bytes: u64) -> Duration {
|
||||
if !self.enable_dynamic_timeout {
|
||||
return self.base_timeout;
|
||||
}
|
||||
|
||||
let mb = size_bytes as f64 / (1024.0 * 1024.0);
|
||||
let timeout = self.base_timeout + self.timeout_per_mb.mul_f64(mb);
|
||||
timeout.clamp(self.min_timeout, self.max_timeout)
|
||||
}
|
||||
|
||||
/// Validate the configuration.
|
||||
pub fn validate(&self) -> Result<(), TimeoutError> {
|
||||
if self.min_timeout > self.max_timeout {
|
||||
return Err(TimeoutError::InvalidConfig("min_timeout must be <= max_timeout".to_string()));
|
||||
}
|
||||
if self.base_timeout < self.min_timeout || self.base_timeout > self.max_timeout {
|
||||
return Err(TimeoutError::InvalidConfig(
|
||||
"base_timeout must be between min_timeout and max_timeout".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Timeout error.
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub enum TimeoutError {
|
||||
/// Operation timed out.
|
||||
#[error("Operation timed out after {0:?}")]
|
||||
TimedOut(Duration),
|
||||
/// Invalid configuration.
|
||||
#[error("Invalid timeout config: {0}")]
|
||||
InvalidConfig(String),
|
||||
}
|
||||
|
||||
/// Operation progress tracker.
|
||||
#[derive(Debug)]
|
||||
pub struct OperationProgress {
|
||||
/// Total size (if known).
|
||||
pub total_size: Option<u64>,
|
||||
/// Bytes processed.
|
||||
bytes_processed: AtomicU64,
|
||||
/// Last update time.
|
||||
last_update: std::sync::Mutex<Instant>,
|
||||
/// Stale timeout.
|
||||
stale_timeout: Duration,
|
||||
/// Start time for transfer rate calculation.
|
||||
start_time: Instant,
|
||||
}
|
||||
|
||||
impl OperationProgress {
|
||||
/// Create new operation progress.
|
||||
pub fn new(total_size: Option<u64>, stale_timeout: Duration) -> Self {
|
||||
Self {
|
||||
total_size,
|
||||
bytes_processed: AtomicU64::new(0),
|
||||
last_update: std::sync::Mutex::new(Instant::now()),
|
||||
stale_timeout,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Update progress.
|
||||
pub fn update(&self, bytes: u64) {
|
||||
self.bytes_processed.store(bytes, Ordering::Relaxed);
|
||||
if let Ok(mut last) = self.last_update.lock() {
|
||||
*last = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Add to progress.
|
||||
pub fn add(&self, bytes: u64) {
|
||||
self.bytes_processed.fetch_add(bytes, Ordering::Relaxed);
|
||||
if let Ok(mut last) = self.last_update.lock() {
|
||||
*last = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Get current progress.
|
||||
pub fn current(&self) -> u64 {
|
||||
self.bytes_processed.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Check if progress is stale.
|
||||
pub fn is_stale(&self) -> bool {
|
||||
if let Ok(last) = self.last_update.lock() {
|
||||
last.elapsed() > self.stale_timeout
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Get progress percentage.
|
||||
pub fn progress_percent(&self) -> Option<f64> {
|
||||
self.total_size.map(|total| {
|
||||
if total == 0 {
|
||||
100.0
|
||||
} else {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
(processed as f64 / total as f64 * 100.0).min(100.0)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Get remaining bytes.
|
||||
pub fn remaining(&self) -> Option<u64> {
|
||||
self.total_size.map(|total| {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
total.saturating_sub(processed)
|
||||
})
|
||||
}
|
||||
|
||||
/// Calculate transfer rate in bytes per second.
|
||||
///
|
||||
/// Returns 0 if no time has elapsed or no data transferred.
|
||||
pub fn transfer_rate(&self) -> u64 {
|
||||
let processed = self.bytes_processed.load(Ordering::Relaxed);
|
||||
if processed == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let elapsed = self.start_time.elapsed().as_secs_f64();
|
||||
if elapsed > 0.0 {
|
||||
(processed as f64 / elapsed) as u64
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Request timeout wrapper.
|
||||
pub struct RequestTimeoutWrapper {
|
||||
/// Configuration.
|
||||
config: TimeoutConfig,
|
||||
/// Start time.
|
||||
start_time: Instant,
|
||||
/// Operation progress.
|
||||
progress: Option<OperationProgress>,
|
||||
}
|
||||
|
||||
impl RequestTimeoutWrapper {
|
||||
/// Create a new timeout wrapper.
|
||||
pub fn new(config: TimeoutConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
start_time: Instant::now(),
|
||||
progress: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create with progress tracking.
|
||||
pub fn with_progress(config: TimeoutConfig, total_size: Option<u64>, stale_timeout: Duration) -> Self {
|
||||
Self {
|
||||
config,
|
||||
start_time: Instant::now(),
|
||||
progress: Some(OperationProgress::new(total_size, stale_timeout)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the configuration.
|
||||
pub fn config(&self) -> &TimeoutConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Get elapsed time.
|
||||
pub fn elapsed(&self) -> Duration {
|
||||
self.start_time.elapsed()
|
||||
}
|
||||
|
||||
/// Get remaining time.
|
||||
pub fn remaining(&self, timeout: Duration) -> Option<Duration> {
|
||||
let elapsed = self.elapsed();
|
||||
if elapsed >= timeout { None } else { Some(timeout - elapsed) }
|
||||
}
|
||||
|
||||
/// Check if timed out.
|
||||
pub fn is_timed_out(&self, size: Option<u64>) -> bool {
|
||||
let timeout = self.get_timeout(size);
|
||||
self.elapsed() > timeout
|
||||
}
|
||||
|
||||
/// Get the timeout for a given size.
|
||||
pub fn get_timeout(&self, size: Option<u64>) -> Duration {
|
||||
if self.config.enable_dynamic_timeout {
|
||||
if let Some(s) = size {
|
||||
self.config.calculate_timeout(s)
|
||||
} else {
|
||||
self.config.base_timeout
|
||||
}
|
||||
} else {
|
||||
self.config.base_timeout
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if timed out and return error if so.
|
||||
pub fn check_timeout(&self, size: Option<u64>) -> Result<(), TimeoutError> {
|
||||
if self.is_timed_out(size) {
|
||||
Err(TimeoutError::TimedOut(self.get_timeout(size)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Get progress.
|
||||
pub fn progress(&self) -> Option<&OperationProgress> {
|
||||
self.progress.as_ref()
|
||||
}
|
||||
|
||||
/// Update progress.
|
||||
pub fn update_progress(&self, bytes: u64) {
|
||||
if let Some(ref progress) = self.progress {
|
||||
progress.update(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if operation is stalled (no progress for a while).
|
||||
pub fn is_stalled(&self) -> bool {
|
||||
self.progress.as_ref().is_some_and(|p| p.is_stale())
|
||||
}
|
||||
|
||||
/// Get progress percentage.
|
||||
pub fn progress_percent(&self) -> Option<f64> {
|
||||
self.progress.as_ref().and_then(|p| p.progress_percent())
|
||||
}
|
||||
}
|
||||
|
||||
/// Timeout statistics.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct TimeoutStats {
|
||||
/// Total operations.
|
||||
pub total_operations: AtomicU64,
|
||||
/// Timed out operations.
|
||||
pub timed_out: AtomicU64,
|
||||
/// Total wait time in nanoseconds.
|
||||
pub total_wait_time_ns: AtomicU64,
|
||||
/// Maximum wait time in nanoseconds.
|
||||
pub max_wait_time_ns: AtomicU64,
|
||||
}
|
||||
|
||||
impl TimeoutStats {
|
||||
/// Create new timeout statistics.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Record an operation.
|
||||
pub fn record_operation(&self, wait_time: Duration) {
|
||||
self.total_operations.fetch_add(1, Ordering::Relaxed);
|
||||
let ns = wait_time.as_nanos() as u64;
|
||||
self.total_wait_time_ns.fetch_add(ns, Ordering::Relaxed);
|
||||
|
||||
let mut current = self.max_wait_time_ns.load(Ordering::Relaxed);
|
||||
while ns > current {
|
||||
match self
|
||||
.max_wait_time_ns
|
||||
.compare_exchange_weak(current, ns, Ordering::Relaxed, Ordering::Relaxed)
|
||||
{
|
||||
Ok(_) => break,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a timeout.
|
||||
pub fn record_timeout(&self) {
|
||||
self.timed_out.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Get timeout rate.
|
||||
pub fn timeout_rate(&self) -> f64 {
|
||||
let total = self.total_operations.load(Ordering::Relaxed);
|
||||
let timed_out = self.timed_out.load(Ordering::Relaxed);
|
||||
if total == 0 { 0.0 } else { timed_out as f64 / total as f64 }
|
||||
}
|
||||
|
||||
/// Get average wait time.
|
||||
pub fn avg_wait_time(&self) -> Duration {
|
||||
let total = self.total_wait_time_ns.load(Ordering::Relaxed);
|
||||
let count = self.total_operations.load(Ordering::Relaxed);
|
||||
total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO)
|
||||
}
|
||||
|
||||
/// Reset statistics.
|
||||
pub fn reset(&self) {
|
||||
self.total_operations.store(0, Ordering::Relaxed);
|
||||
self.timed_out.store(0, Ordering::Relaxed);
|
||||
self.total_wait_time_ns.store(0, Ordering::Relaxed);
|
||||
self.max_wait_time_ns.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate adaptive timeout based on historical data and current conditions.
|
||||
///
|
||||
/// This function adjusts the timeout based on:
|
||||
/// - Historical transfer rate
|
||||
/// - Recent timeout count
|
||||
/// - Object size
|
||||
pub fn calculate_adaptive_timeout(
|
||||
base_timeout: Duration,
|
||||
historical_rate_bps: Option<u64>,
|
||||
recent_timeout_count: u32,
|
||||
object_size: u64,
|
||||
) -> Duration {
|
||||
// If we have recent timeouts, increase timeout
|
||||
let timeout_multiplier = if recent_timeout_count > 3 {
|
||||
2.0 // Double timeout if many recent timeouts
|
||||
} else if recent_timeout_count > 1 {
|
||||
1.5 // 50% increase if some timeouts
|
||||
} else {
|
||||
1.0 // No adjustment
|
||||
};
|
||||
|
||||
// Adaptive timeout bounds: 5 seconds minimum, 10 minutes maximum.
|
||||
const MIN_SECS: f64 = 5.0;
|
||||
const MAX_SECS: f64 = 600.0;
|
||||
|
||||
// If we have historical rate data, use it for estimation
|
||||
let estimated_secs = match historical_rate_bps {
|
||||
Some(rate) if rate > 0 => (object_size as f64 / rate as f64) * 1.2, // 20% buffer
|
||||
_ => base_timeout.as_secs_f64(),
|
||||
};
|
||||
|
||||
// Clamp BEFORE constructing the Duration: `from_secs_f64` panics when the
|
||||
// estimate overflows Duration (huge object_size with a tiny historical rate).
|
||||
Duration::from_secs_f64((estimated_secs * timeout_multiplier).clamp(MIN_SECS, MAX_SECS))
|
||||
}
|
||||
|
||||
/// Estimate bytes per second transfer rate.
|
||||
///
|
||||
/// This is used for adaptive timeout calculation.
|
||||
pub fn estimate_bytes_per_second(object_size: u64, expected_duration: Duration) -> u64 {
|
||||
let secs = expected_duration.as_secs_f64();
|
||||
if secs > 0.0 {
|
||||
(object_size as f64 / secs) as u64
|
||||
} else {
|
||||
// Return a reasonable default (1 MB/s)
|
||||
1024 * 1024
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_timeout_config() {
|
||||
let config = TimeoutConfig::default();
|
||||
assert!(config.validate().is_ok());
|
||||
|
||||
// Small file
|
||||
let timeout = config.calculate_timeout(1024);
|
||||
assert!(timeout >= config.min_timeout);
|
||||
|
||||
// Large file
|
||||
let timeout = config.calculate_timeout(100 * 1024 * 1024);
|
||||
assert!(timeout <= config.max_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timeout_config_validation() {
|
||||
let config = TimeoutConfig {
|
||||
min_timeout: Duration::from_secs(10),
|
||||
max_timeout: Duration::from_secs(5),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adaptive_timeout_extreme_estimate_does_not_panic() {
|
||||
// A huge object with a tiny historical rate used to overflow
|
||||
// Duration::from_secs_f64 and panic; it must clamp to the upper bound.
|
||||
let timeout = calculate_adaptive_timeout(Duration::from_secs(30), Some(1), 0, u64::MAX);
|
||||
assert_eq!(timeout, Duration::from_secs(600));
|
||||
|
||||
// Tiny estimates clamp to the lower bound.
|
||||
let timeout = calculate_adaptive_timeout(Duration::from_secs(30), Some(u64::MAX), 0, 1);
|
||||
assert_eq!(timeout, Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operation_progress() {
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
|
||||
assert_eq!(progress.current(), 0);
|
||||
assert_eq!(progress.progress_percent(), Some(0.0));
|
||||
|
||||
progress.update(500);
|
||||
assert_eq!(progress.current(), 500);
|
||||
assert_eq!(progress.progress_percent(), Some(50.0));
|
||||
|
||||
progress.add(300);
|
||||
assert_eq!(progress.current(), 800);
|
||||
assert_eq!(progress.remaining(), Some(200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_request_timeout_wrapper() {
|
||||
let config = TimeoutConfig {
|
||||
base_timeout: Duration::from_millis(100),
|
||||
enable_dynamic_timeout: false,
|
||||
..Default::default()
|
||||
};
|
||||
let wrapper = RequestTimeoutWrapper::new(config);
|
||||
|
||||
assert!(!wrapper.is_timed_out(None));
|
||||
|
||||
std::thread::sleep(Duration::from_millis(150));
|
||||
|
||||
assert!(wrapper.is_timed_out(None));
|
||||
assert!(wrapper.check_timeout(None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_timeout_stats() {
|
||||
let stats = TimeoutStats::new();
|
||||
|
||||
stats.record_operation(Duration::from_millis(10));
|
||||
stats.record_operation(Duration::from_millis(20));
|
||||
stats.record_timeout();
|
||||
|
||||
assert_eq!(stats.total_operations.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(stats.timed_out.load(Ordering::Relaxed), 1);
|
||||
assert!((stats.timeout_rate() - 0.5).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_progress_tracking() {
|
||||
let config = TimeoutConfig::default();
|
||||
let wrapper = RequestTimeoutWrapper::with_progress(config, Some(1000), Duration::from_secs(1));
|
||||
|
||||
wrapper.update_progress(500);
|
||||
assert_eq!(wrapper.progress_percent(), Some(50.0));
|
||||
assert!(!wrapper.is_stalled());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! BytesMut-backed object writer for optimized write operations.
|
||||
//!
|
||||
//! It uses `BytesMut` for efficient buffering; writes into that buffer may
|
||||
//! still copy input bytes. The historical `ZeroCopyObjectWriter` name remains
|
||||
//! available as a deprecated compatibility alias.
|
||||
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
/// BytesMut-backed object writer for optimized write operations.
|
||||
///
|
||||
/// This writer minimizes memory allocations by:
|
||||
/// - Using BytesMut for efficient buffer growth
|
||||
/// - Accepting `Bytes` inputs for efficient buffer handling
|
||||
/// - Optional integration with BytesPool for buffer reuse
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use rustfs_io_core::BytesMutWriter;
|
||||
/// use bytes::Bytes;
|
||||
///
|
||||
/// #[tokio::main]
|
||||
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let mut writer = BytesMutWriter::new();
|
||||
///
|
||||
/// // Write into the internal BytesMut buffer
|
||||
/// let data = Bytes::from("hello world");
|
||||
/// writer.write_buffered(data).await?;
|
||||
///
|
||||
/// // Get the result as Bytes (zero-copy conversion)
|
||||
/// let result = writer.into_bytes();
|
||||
///
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
pub struct BytesMutWriter {
|
||||
/// Internal buffer using BytesMut for efficient growth
|
||||
buffer: BytesMut,
|
||||
/// Total bytes written
|
||||
bytes_written: usize,
|
||||
/// Whether the writer has been finalized
|
||||
finalized: bool,
|
||||
}
|
||||
|
||||
/// Historical name for the BytesMut-backed object writer.
|
||||
#[deprecated(since = "1.0.0-beta.8", note = "use BytesMutWriter; writes append into a BytesMut buffer")]
|
||||
pub type ZeroCopyObjectWriter = BytesMutWriter;
|
||||
|
||||
impl BytesMutWriter {
|
||||
/// Create a new bytes-backed object writer with default capacity (8KB).
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let writer = BytesMutWriter::new();
|
||||
/// ```
|
||||
pub fn new() -> Self {
|
||||
Self::with_capacity(8 * 1024)
|
||||
}
|
||||
|
||||
/// Create a new bytes-backed object writer with specified capacity.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `capacity` - Initial buffer capacity in bytes
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let writer = BytesMutWriter::with_capacity(64 * 1024);
|
||||
/// ```
|
||||
pub fn with_capacity(capacity: usize) -> Self {
|
||||
Self {
|
||||
buffer: BytesMut::with_capacity(capacity),
|
||||
bytes_written: 0,
|
||||
finalized: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write data into the internal buffer.
|
||||
///
|
||||
/// This method accepts `Bytes` for API compatibility, then appends the
|
||||
/// bytes into the internal `BytesMut` buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data` - Data to append to the internal buffer
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * `Ok(usize)` - Number of bytes written
|
||||
/// * `Err(ZeroCopyWriteError)` - Write error
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let data = Bytes::from("hello world");
|
||||
/// let written = writer.write_buffered(data).await?;
|
||||
/// ```
|
||||
pub async fn write_buffered(&mut self, data: Bytes) -> Result<usize, ZeroCopyWriteError> {
|
||||
if self.finalized {
|
||||
return Err(ZeroCopyWriteError::Finalized("Cannot write to finalized writer".to_string()));
|
||||
}
|
||||
|
||||
let len = data.len();
|
||||
self.buffer.put(data);
|
||||
|
||||
self.bytes_written += len;
|
||||
Ok(len)
|
||||
}
|
||||
|
||||
/// Historical name for `write_buffered`.
|
||||
#[deprecated(
|
||||
since = "1.0.0-beta.8",
|
||||
note = "use write_buffered; this method appends bytes into an internal buffer"
|
||||
)]
|
||||
pub async fn write_zero_copy(&mut self, data: Bytes) -> Result<usize, ZeroCopyWriteError> {
|
||||
self.write_buffered(data).await
|
||||
}
|
||||
|
||||
/// Write a slice of data.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `data` - Data slice to write
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * `Ok(usize)` - Number of bytes written
|
||||
/// * `Err(ZeroCopyWriteError)` - Write error
|
||||
pub async fn write_slice(&mut self, data: &[u8]) -> Result<usize, ZeroCopyWriteError> {
|
||||
if self.finalized {
|
||||
return Err(ZeroCopyWriteError::Finalized("Cannot write to finalized writer".to_string()));
|
||||
}
|
||||
|
||||
let len = data.len();
|
||||
self.buffer.put_slice(data);
|
||||
self.bytes_written += len;
|
||||
Ok(len)
|
||||
}
|
||||
|
||||
/// Finalize the writer and consume it, returning the written data as Bytes.
|
||||
///
|
||||
/// This converts the internal BytesMut to Bytes, which is a zero-copy
|
||||
/// operation that freezes the buffer.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The written data as Bytes
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let result = writer.into_bytes();
|
||||
/// ```
|
||||
pub fn into_bytes(mut self) -> Bytes {
|
||||
self.finalized = true;
|
||||
self.buffer.freeze()
|
||||
}
|
||||
|
||||
/// Get the current buffer as a slice (without consuming).
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Slice of the current buffer content
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
&self.buffer[..]
|
||||
}
|
||||
|
||||
/// Get the total number of bytes written.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Number of bytes written
|
||||
pub fn bytes_written(&self) -> usize {
|
||||
self.bytes_written
|
||||
}
|
||||
|
||||
/// Get the current buffer capacity.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Current buffer capacity in bytes
|
||||
pub fn capacity(&self) -> usize {
|
||||
self.buffer.capacity()
|
||||
}
|
||||
|
||||
/// Get the current buffer length.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Current buffer length in bytes
|
||||
pub fn len(&self) -> usize {
|
||||
self.buffer.len()
|
||||
}
|
||||
|
||||
/// Check if the buffer is empty.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` if buffer is empty, `false` otherwise
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.buffer.is_empty()
|
||||
}
|
||||
|
||||
/// Clear the buffer, resetting it to empty.
|
||||
///
|
||||
/// This does not change the capacity, just resets the length to 0.
|
||||
pub fn clear(&mut self) {
|
||||
self.buffer.clear();
|
||||
self.bytes_written = 0;
|
||||
self.finalized = false;
|
||||
}
|
||||
|
||||
/// Reserve additional capacity in the buffer.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `additional` - Additional capacity to reserve
|
||||
pub fn reserve(&mut self, additional: usize) {
|
||||
self.buffer.reserve(additional);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BytesMutWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BytesMutWriter {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("BytesMutWriter")
|
||||
.field("buffer_len", &self.buffer.len())
|
||||
.field("buffer_capacity", &self.buffer.capacity())
|
||||
.field("bytes_written", &self.bytes_written)
|
||||
.field("finalized", &self.finalized)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// AsyncWrite implementation for BytesMutWriter.
|
||||
///
|
||||
/// This allows the writer to be used with tokio's async I/O utilities.
|
||||
impl AsyncWrite for BytesMutWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize, tokio::io::Error>> {
|
||||
if self.finalized {
|
||||
return Poll::Ready(Err(tokio::io::Error::new(
|
||||
tokio::io::ErrorKind::WriteZero,
|
||||
"Cannot write to finalized writer",
|
||||
)));
|
||||
}
|
||||
|
||||
let len = buf.len();
|
||||
self.buffer.put_slice(buf);
|
||||
self.bytes_written += len;
|
||||
Poll::Ready(Ok(len))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), tokio::io::Error>> {
|
||||
// Nothing to flush for in-memory buffer
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), tokio::io::Error>> {
|
||||
self.finalized = true;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Zero-copy write error types.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ZeroCopyWriteError {
|
||||
/// I/O error occurred
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] tokio::io::Error),
|
||||
|
||||
/// Writer has been finalized and cannot accept more writes
|
||||
#[error("Writer finalized: {0}")]
|
||||
Finalized(String),
|
||||
|
||||
/// Invalid input provided
|
||||
#[error("Invalid input: {0}")]
|
||||
InvalidInput(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_writer() {
|
||||
let writer = BytesMutWriter::new();
|
||||
assert!(writer.is_empty());
|
||||
assert_eq!(writer.bytes_written(), 0);
|
||||
assert!(writer.capacity() >= 8 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_buffered() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let data = Bytes::from("hello world");
|
||||
|
||||
let written = writer.write_buffered(data).await.unwrap();
|
||||
assert_eq!(written, 11);
|
||||
assert_eq!(writer.bytes_written(), 11);
|
||||
assert_eq!(writer.as_slice(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_preferred_writer_alias() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let written = writer
|
||||
.write_buffered(Bytes::from("hello world"))
|
||||
.await
|
||||
.expect("write bytes through alias");
|
||||
|
||||
assert_eq!(written, 11);
|
||||
assert_eq!(writer.as_slice(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_slice() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let data = b"hello world";
|
||||
|
||||
let written = writer.write_slice(data).await.unwrap();
|
||||
assert_eq!(written, 11);
|
||||
assert_eq!(writer.bytes_written(), 11);
|
||||
assert_eq!(writer.as_slice(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_into_bytes() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let data = Bytes::from("hello world");
|
||||
|
||||
writer.write_buffered(data).await.unwrap();
|
||||
let result = writer.into_bytes();
|
||||
|
||||
assert_eq!(result.as_ref(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_after_finalize() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let data = Bytes::from("hello");
|
||||
|
||||
writer.write_buffered(data).await.unwrap();
|
||||
let _result = writer.into_bytes();
|
||||
|
||||
// Create new writer and try to write after finalize
|
||||
let mut writer2 = BytesMutWriter::new();
|
||||
writer2.write_buffered(Bytes::from("test")).await.unwrap();
|
||||
let _ = writer2.into_bytes();
|
||||
|
||||
// Writing to a consumed writer should work via new writer
|
||||
let mut writer3 = BytesMutWriter::new();
|
||||
let result = writer3.write_buffered(Bytes::from("final")).await;
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clear() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
writer.write_slice(b"hello").await.unwrap();
|
||||
|
||||
writer.clear();
|
||||
assert!(writer.is_empty());
|
||||
assert_eq!(writer.bytes_written(), 0);
|
||||
// Capacity should remain
|
||||
assert!(writer.capacity() > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reserve() {
|
||||
let mut writer = BytesMutWriter::with_capacity(10);
|
||||
let initial_capacity = writer.capacity();
|
||||
|
||||
writer.reserve(1000);
|
||||
// Reserve ensures at least the additional capacity can be added
|
||||
// but may allocate more than requested
|
||||
assert!(writer.capacity() >= initial_capacity);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_writes() {
|
||||
let mut writer = BytesMutWriter::new();
|
||||
|
||||
writer.write_buffered(Bytes::from("hello ")).await.unwrap();
|
||||
writer.write_slice(b"world").await.unwrap();
|
||||
|
||||
assert_eq!(writer.as_slice(), b"hello world");
|
||||
assert_eq!(writer.bytes_written(), 11);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_async_write() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let mut writer = BytesMutWriter::new();
|
||||
let data = b"hello world";
|
||||
|
||||
let written = writer.write(data).await.unwrap();
|
||||
assert_eq!(written, 11);
|
||||
assert_eq!(writer.as_slice(), b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_debug() {
|
||||
let writer = BytesMutWriter::new();
|
||||
let debug_str = format!("{:?}", writer);
|
||||
assert!(debug_str.contains("BytesMutWriter"));
|
||||
assert!(debug_str.contains("buffer_len"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(deprecated)]
|
||||
async fn test_legacy_writer_alias() {
|
||||
let mut writer = ZeroCopyObjectWriter::new();
|
||||
let written = writer.write_zero_copy(Bytes::from("hello")).await.unwrap();
|
||||
|
||||
assert_eq!(written, 5);
|
||||
assert_eq!(writer.as_slice(), b"hello");
|
||||
}
|
||||
}
|
||||
@@ -293,15 +293,6 @@ enum StrictVaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
Kubernetes {
|
||||
role: String,
|
||||
#[serde(default)]
|
||||
mount: Option<String>,
|
||||
#[serde(default)]
|
||||
jwt_path: Option<std::path::PathBuf>,
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
TokenFile {
|
||||
path: std::path::PathBuf,
|
||||
#[serde(default)]
|
||||
@@ -328,17 +319,6 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => Self::Kubernetes {
|
||||
role,
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -519,7 +499,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -534,7 +513,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -923,42 +901,6 @@ mod tests {
|
||||
assert!(request.to_kms_config().validate().is_ok());
|
||||
}
|
||||
|
||||
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||
/// manifest carries no credential and no cluster-specific paths.
|
||||
#[test]
|
||||
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||
let raw = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"mount_path": "rustfs",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||
});
|
||||
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||
let config = request.to_kms_config();
|
||||
config.validate().expect("kubernetes auth must validate");
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
|
||||
let unknown_field = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||
});
|
||||
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||
|
||||
@@ -550,7 +550,6 @@ impl VaultKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -326,97 +326,6 @@ impl fmt::Debug for AppRoleLogin {
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||
///
|
||||
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||
///
|
||||
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||
/// the projected token and mounts it world-readable by default, so rejecting
|
||||
/// group/other bits would refuse every standard pod rather than catch a
|
||||
/// deployment error.
|
||||
pub(crate) struct KubernetesLogin {
|
||||
/// Unauthenticated client used only for the login exchange.
|
||||
login_client: VaultClient,
|
||||
mount: String,
|
||||
role: String,
|
||||
jwt_path: PathBuf,
|
||||
}
|
||||
|
||||
impl KubernetesLogin {
|
||||
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||
Ok(Self {
|
||||
login_client: settings.build_login_client()?,
|
||||
mount,
|
||||
role,
|
||||
jwt_path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the ServiceAccount token for one login attempt.
|
||||
///
|
||||
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||
/// has not projected yet heals the source without a restart.
|
||||
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||
.await
|
||||
.map_err(|error| AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
})?;
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
raw.zeroize();
|
||||
return Err(AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Kubernetes ServiceAccount token {} is empty",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
});
|
||||
}
|
||||
let jwt = SecretString::new(trimmed.to_string());
|
||||
raw.zeroize();
|
||||
Ok(jwt)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TokenSource for KubernetesLogin {
|
||||
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||
let jwt = self.resolve_jwt().await?;
|
||||
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||
.await
|
||||
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
|
||||
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||
let auth = vaultrs::token::renew_self(client, None)
|
||||
.await
|
||||
.map_err(|error| attempt_error("token renewal", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for KubernetesLogin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// The login client embeds Vault client settings and must stay out of
|
||||
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||
f.debug_struct("KubernetesLogin")
|
||||
.field("mount", &self.mount)
|
||||
.field("role", &self.role)
|
||||
.field("jwt_path", &self.jwt_path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||
/// token file (for example a Vault Agent auto-auth sink).
|
||||
///
|
||||
@@ -555,9 +464,6 @@ pub(crate) fn token_source_for(
|
||||
secret_id.clone(),
|
||||
secret_id_file.clone(),
|
||||
)?)),
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -580,9 +486,6 @@ pub(crate) struct VaultConnectionSettings {
|
||||
pub(crate) namespace: Option<String>,
|
||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||
pub(crate) attempt_timeout: Duration,
|
||||
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||
pub(crate) skip_tls_verify: bool,
|
||||
}
|
||||
|
||||
impl VaultConnectionSettings {
|
||||
@@ -596,11 +499,6 @@ impl VaultConnectionSettings {
|
||||
// operation-level retry policy.
|
||||
settings_builder.timeout(Some(self.attempt_timeout));
|
||||
settings_builder.token(token);
|
||||
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||
// disable certificate verification behind the KMS configuration and its
|
||||
// insecure-defaults gate.
|
||||
settings_builder.verify(!self.skip_tls_verify);
|
||||
|
||||
if let Some(namespace) = &self.namespace {
|
||||
settings_builder.namespace(Some(namespace.clone()));
|
||||
@@ -653,10 +551,6 @@ impl VaultCredentialPolicy {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::Kubernetes {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::TokenFile {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
@@ -690,25 +584,15 @@ pub(crate) struct VaultClientHandle {
|
||||
|
||||
impl VaultClientHandle {
|
||||
/// Absolute expiry of this generation's token.
|
||||
///
|
||||
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||
/// collapses to `None` — the same answer already given for the zero-lease
|
||||
/// tokens Vault issues, which keeps the token in use and still fully
|
||||
/// validated by Vault on every call.
|
||||
fn expires_at(&self) -> Option<Instant> {
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
||||
}
|
||||
|
||||
/// When the renewal task should refresh this generation: half the TTL,
|
||||
/// leaving the second half as budget for retries before the fail-closed
|
||||
/// window is reached.
|
||||
///
|
||||
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||
/// leaving a token that never expires with nothing to renew.
|
||||
fn renew_at(&self) -> Option<Instant> {
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -778,7 +662,7 @@ impl VaultCredentialProvider {
|
||||
let handle = self.current.load_full();
|
||||
if let Some(expires_at) = handle.expires_at() {
|
||||
let now = Instant::now();
|
||||
if self.inside_safety_window(now, expires_at) {
|
||||
if now + self.policy.safety_window >= expires_at {
|
||||
return Err(KmsError::credentials_unavailable(format!(
|
||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||
handle.generation, self.policy.safety_window
|
||||
@@ -788,18 +672,6 @@ impl VaultCredentialProvider {
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||
///
|
||||
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||
/// guaranteed to have passed this version's validation: a window too large
|
||||
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||
/// means every token is always inside it, so saturating to "refuse" is both
|
||||
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||
now.checked_add(self.policy.safety_window)
|
||||
.is_none_or(|deadline| deadline >= expires_at)
|
||||
}
|
||||
|
||||
/// Publish the credential gauges for the generation currently installed.
|
||||
///
|
||||
/// The fail-closed gauge re-evaluates the very gate
|
||||
@@ -811,7 +683,7 @@ impl VaultCredentialProvider {
|
||||
let fail_closed = match handle.expires_at() {
|
||||
Some(expires_at) => {
|
||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||
self.inside_safety_window(now, expires_at)
|
||||
now + self.policy.safety_window >= expires_at
|
||||
}
|
||||
// A generation without an expiry has no remaining TTL to report
|
||||
// and can never lapse, so it can never fail closed either.
|
||||
@@ -988,7 +860,7 @@ impl Drop for CredentialTaskHandle {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||
use crate::config::REDACTED_SECRET;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
|
||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||
@@ -999,7 +871,6 @@ mod tests {
|
||||
address: "http://127.0.0.1:8200".to_string(),
|
||||
namespace: Some("team-namespace".to_string()),
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1186,143 +1057,6 @@ mod tests {
|
||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||
let settings = test_settings();
|
||||
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||
.expect("kubernetes auth must map to a login source");
|
||||
|
||||
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||
}
|
||||
|
||||
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||
/// path from persisted configuration, so the fail-closed comparison must
|
||||
/// survive a window too large to add to the current instant. Before the
|
||||
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||
/// instant" on the first request after a lease-bearing login.
|
||||
#[tokio::test]
|
||||
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = provider
|
||||
.current()
|
||||
.expect_err("a window wider than any lease must refuse the token");
|
||||
assert!(
|
||||
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||
"expected CredentialsUnavailable, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||
/// the other side of the same comparison, so an absurd one must not panic
|
||||
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||
/// the zero-lease case already behaves.
|
||||
#[tokio::test]
|
||||
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(u64::MAX),
|
||||
true,
|
||||
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
provider
|
||||
.current()
|
||||
.expect("a token whose expiry cannot be represented must stay usable");
|
||||
}
|
||||
|
||||
/// The configured flag has to reach the HTTP client, not just the config
|
||||
/// struct: every generation (authenticated and login) builds its own client,
|
||||
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||
/// each one carries the setting.
|
||||
#[test]
|
||||
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||
for skip_tls_verify in [false, true] {
|
||||
let settings = VaultConnectionSettings {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
namespace: None,
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify,
|
||||
};
|
||||
|
||||
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||
|
||||
let login = settings.build_login_client().expect("login client must build");
|
||||
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||
}
|
||||
}
|
||||
|
||||
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||
/// builder leaves it unset, which would disable certificate verification
|
||||
/// without passing the KMS insecure-defaults gate.
|
||||
#[test]
|
||||
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||
assert!(
|
||||
client.settings.verify,
|
||||
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||
/// does not corrupt the assertion sent to Vault.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("token");
|
||||
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||
|
||||
let login = KubernetesLogin::new(
|
||||
&test_settings(),
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
"rustfs".to_string(),
|
||||
path.clone(),
|
||||
)
|
||||
.expect("login source must build");
|
||||
|
||||
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||
|
||||
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||
assert_eq!(
|
||||
login.resolve_jwt().await.expect("second read").expose(),
|
||||
"rotated-jwt",
|
||||
"a rotated projected token must be picked up without a restart"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||
/// keeps retrying, which is what lets a late projection heal the source.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let missing = dir.path().join("absent-token");
|
||||
let empty = dir.path().join("empty-token");
|
||||
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||
|
||||
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||
let login =
|
||||
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||
.expect("login source must build");
|
||||
|
||||
let error = login
|
||||
.acquire()
|
||||
.await
|
||||
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_renewal_task_renews_at_half_ttl() {
|
||||
let (provider, state) = scripted_provider(
|
||||
|
||||
@@ -415,7 +415,6 @@ impl VaultTransitKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -450,10 +450,6 @@ impl VaultRestoreClient {
|
||||
address: target.address.clone(),
|
||||
namespace: target.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
// A restore target carries no TLS settings, so certificates are
|
||||
// always verified: recovery is the last path that should accept an
|
||||
// unauthenticated Vault.
|
||||
skip_tls_verify: false,
|
||||
};
|
||||
let source = token_source_for(&target.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
+78
-299
@@ -25,10 +25,6 @@ use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -39,9 +35,6 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||
@@ -52,9 +45,6 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
|
||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||
///
|
||||
@@ -94,14 +84,6 @@ fn default_vault_approle_mount() -> String {
|
||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_mount() -> String {
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||
}
|
||||
|
||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||
@@ -508,23 +490,6 @@ pub enum VaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||
/// for a lease-bound Vault token that is renewed in the background.
|
||||
Kubernetes {
|
||||
/// Vault role bound to this ServiceAccount.
|
||||
role: String,
|
||||
/// Kubernetes auth engine mount path.
|
||||
#[serde(default = "default_vault_kubernetes_mount")]
|
||||
mount: String,
|
||||
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||
/// a token the kubelet rotates is picked up without a restart.
|
||||
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||
jwt_path: PathBuf,
|
||||
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||
/// per-attempt timeout.
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||
/// the token is read from `path` and re-read periodically so a token
|
||||
/// rotated by the agent is picked up without a restart.
|
||||
@@ -555,16 +520,6 @@ impl VaultAuthMethod {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kubernetes authentication with the default mount and projected token path.
|
||||
pub fn kubernetes(role: String) -> Self {
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount: default_vault_kubernetes_mount(),
|
||||
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||
refresh_safety_window_secs: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent-managed token file with the default poll interval.
|
||||
pub fn token_file(path: PathBuf) -> Self {
|
||||
Self::TokenFile {
|
||||
@@ -593,20 +548,6 @@ impl fmt::Debug for VaultAuthMethod {
|
||||
.field("mount", mount)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
// No redaction: the role and mount name a Vault binding, and the
|
||||
// ServiceAccount token itself is never held on this type.
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => f
|
||||
.debug_struct("Kubernetes")
|
||||
.field("role", role)
|
||||
.field("mount", mount)
|
||||
.field("jwt_path", jwt_path)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
Self::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1087,12 +1028,50 @@ impl KmsConfig {
|
||||
});
|
||||
}
|
||||
KmsBackend::VaultKv2 => {
|
||||
config.backend_config =
|
||||
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
}
|
||||
KmsBackend::VaultTransit => {
|
||||
config.backend_config =
|
||||
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
||||
metadata_kv_mount: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
),
|
||||
metadata_key_prefix: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
||||
),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
}
|
||||
KmsBackend::Static => {
|
||||
// Read from file first, then fall back to direct env var
|
||||
@@ -1223,78 +1202,6 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
path.starts_with(std::env::temp_dir())
|
||||
}
|
||||
|
||||
/// Command-line values that take precedence over the matching environment
|
||||
/// variables when assembling a Vault backend configuration.
|
||||
///
|
||||
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||
/// reads, so these are only set when the operator passed an explicit flag.
|
||||
///
|
||||
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||
/// error instead of a leak.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct VaultCliOverrides<'a> {
|
||||
pub address: Option<&'a str>,
|
||||
pub token: Option<&'a str>,
|
||||
pub mount_path: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||
///
|
||||
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||
let mount_path = match overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||
{
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
Ok(VaultConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Assemble the Vault Transit backend configuration from the environment.
|
||||
///
|
||||
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||
/// points share it.
|
||||
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||
Ok(VaultTransitConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path: overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||
override_value
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||
}
|
||||
|
||||
/// Resolve the Vault auth method from environment variables.
|
||||
///
|
||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||
@@ -1302,59 +1209,27 @@ fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||
///
|
||||
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||
/// presents the pod's projected ServiceAccount token.
|
||||
///
|
||||
/// `token_override` carries a token supplied on the command line; it stands in
|
||||
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||
/// so a flag and the variable it mirrors select the same method.
|
||||
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||
let token = token_override
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||
|
||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||
// A token file names one authoritative credential source; combining it
|
||||
// with another one would leave the effective identity ambiguous, so
|
||||
// that is a configuration error rather than a precedence rule.
|
||||
for (name, configured) in [
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||
] {
|
||||
if configured {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||
}
|
||||
|
||||
if let Some(role) = kubernetes_role {
|
||||
// Unlike a leftover static token, a second login method is never a
|
||||
// stale remnant: both were configured deliberately and neither can be
|
||||
// ranked over the other.
|
||||
if role_id.is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
return Ok(VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||
refresh_safety_window_secs: None,
|
||||
});
|
||||
}
|
||||
|
||||
let Some(role_id) = role_id else {
|
||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
||||
return Ok(VaultAuthMethod::Token {
|
||||
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1398,22 +1273,6 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => {
|
||||
if role.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||
}
|
||||
if mount.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||
}
|
||||
if jwt_path.as_os_str().is_empty() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1729,10 +1588,30 @@ mod tests {
|
||||
assert!(config.validate().is_ok(), "deprecated mount_path must not be required");
|
||||
}
|
||||
|
||||
// The "VaultKv2 must not claim Transit wrapping" documentation-claim
|
||||
// invariant is enforced by scripts/check_fips_wording.sh, which scans every
|
||||
// file in crates/kms rather than a fixed include_str! list
|
||||
// (rustfs/backlog#1884).
|
||||
#[test]
|
||||
fn test_vault_kv2_sources_do_not_claim_transit_wrapping() {
|
||||
let sources = [
|
||||
("config.rs", include_str!("config.rs")),
|
||||
("api_types.rs", include_str!("api_types.rs")),
|
||||
("backends/vault.rs", include_str!("backends/vault.rs")),
|
||||
("lib.rs", include_str!("lib.rs")),
|
||||
];
|
||||
// Assemble the needles at runtime so this guard does not match its own source.
|
||||
let needles = [
|
||||
format!("wrapping via {}", "Transit"),
|
||||
format!("KV v2 + {}", "Transit"),
|
||||
format!("KV2+{}", "Transit"),
|
||||
format!("you would use Vault's {} engine", "transit"),
|
||||
];
|
||||
for (name, source) in sources {
|
||||
for needle in &needles {
|
||||
assert!(
|
||||
!source.contains(needle.as_str()),
|
||||
"{name} still describes the Vault KV2 backend with `{needle}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_legacy_persisted_vault_transit_config_uses_metadata_defaults() {
|
||||
@@ -2097,106 +1976,6 @@ mod tests {
|
||||
.expect("well-formed token file auth must validate");
|
||||
}
|
||||
|
||||
/// A Kubernetes role alone configures the method: the credential is the
|
||||
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||
/// environment and the mount and token path fall back to the cluster
|
||||
/// defaults.
|
||||
#[test]
|
||||
fn test_from_env_selects_kubernetes() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!(
|
||||
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||
vault.auth_method
|
||||
);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
assert_eq!(refresh_safety_window_secs, &None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
],
|
||||
|| {
|
||||
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||
backend: KmsBackend::VaultTransit,
|
||||
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
auth_method,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes role must be rejected");
|
||||
assert!(error.to_string().contains("role"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: String::new(),
|
||||
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes mount must be rejected");
|
||||
assert!(error.to_string().contains("mount"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
jwt_path: PathBuf::new(),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||
assert!(error.to_string().contains("token path"), "got {error}");
|
||||
|
||||
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||
.validate()
|
||||
.expect("well-formed kubernetes auth must validate");
|
||||
}
|
||||
|
||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||
/// empty one names a path no Vault engine answers. The Transit backend
|
||||
/// already rejects its own empty mounts; this closes the same gap on the
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user